From 5cceff08d79f7497cbb4e713f9e2c8a6f33bf47e Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Fri, 21 Jun 2024 11:27:03 +0530 Subject: [PATCH 001/235] feat: Refactoring the V2 APIs --- api-service/package-lock.json | 24326 ---------------- api-service/src/v1/helpers/Datasets.ts | 6 +- .../DatasetCreateValidationSchema.json | 23 +- 3 files changed, 27 insertions(+), 24328 deletions(-) delete mode 100644 api-service/package-lock.json diff --git a/api-service/package-lock.json b/api-service/package-lock.json deleted file mode 100644 index 833f2f74..00000000 --- a/api-service/package-lock.json +++ /dev/null @@ -1,24326 +0,0 @@ -{ - "name": "obsrv-api-service", - "version": "1.0.0", - "lockfileVersion": 2, - "requires": true, - "packages": { - "": { - "name": "obsrv-api-service", - "version": "1.0.0", - "license": "MIT", - "dependencies": { - "@aws-sdk/client-s3": "^3.540.0", - "@aws-sdk/credential-providers": "^3.309.0", - "@aws-sdk/lib-storage": "^3.182.0", - "@aws-sdk/s3-request-presigner": "^3.540.0", - "@azure/storage-blob": "^12.17.0", - "@google-cloud/storage": "^7.9.0", - "@project-sunbird/logger": "^0.0.9", - "ajv": "^8.11.2", - "ajv-formats": "^2.1.1", - "aws-sdk": "^2.1348.0", - "axios": "^1.6.0", - "body-parser": "^1.20.2", - "compression": "^1.7.4", - "dateformat": "2.0.0", - "express": "^4.18.2", - "http-errors": "^2.0.0", - "http-status": "^1.5.3", - "kafka-node": "^5.0.0", - "kafkajs": "^2.2.4", - "kafkajs-snappy": "^1.1.0", - "kafkajs-snappy-typescript": "^1.0.3", - "knex": "^2.4.2", - "lodash": "^4.17.21", - "log4js": "^6.9.1", - "moment": "^2.29.4", - "multiparty": "4.2.1", - "node-sql-parser": "^5.1.0", - "pg": "^8.11.3", - "pg-hstore": "^2.3.4", - "prom-client": "^14.2.0", - "sequelize": "^6.37.1", - "slug": "^9.0.0", - "trino-client": "^0.2.2", - "uuid": "3.1.0", - "winston": "~2.4.3", - "winston-daily-rotate-file": "~3.2.1" - }, - "devDependencies": { - "@types/chai": "^4.3.3", - "@types/chai-as-promised": "^7.1.5", - "@types/chai-spies": "^1.0.3", - "@types/compression": "^1.7.2", - "@types/express": "^4.17.14", - "@types/http-errors": "^2.0.1", - "@types/kafkajs": "^1.9.0", - "@types/knex": "^0.16.1", - "@types/lodash": "^4.14.190", - "@types/mocha": "^10.0.0", - "@types/mock-knex": "^0.4.4", - "@types/moment": "^2.13.0", - "@types/node": "^18.11.9", - "@types/pg": "^8.6.6", - "@types/sinon": "^17.0.3", - "@types/slug": "^5.0.8", - "@types/uuid": "^9.0.1", - "@typescript-eslint/eslint-plugin": "^7.1.1", - "@typescript-eslint/parser": "^7.1.1", - "chai": "^4.3.6", - "chai-as-promised": "^7.1.1", - "chai-http": "^4.3.0", - "chai-spies": "^1.0.0", - "diff-json": "^2.0.0", - "eslint": "^8.57.0", - "mocha": "^10.1.0", - "nock": "^13.2.9", - "nodemon": "^3.0.1", - "nyc": "^15.1.0", - "sinon": "^17.0.1", - "ts-node": "^10.9.1", - "tsconfig-paths": "^4.1.0", - "typescript": "^4.8.4" - } - }, - "../../config-service-ext/config-service/dist": { - "name": "obsrv-config-service-ext", - "version": "1.0.0", - "extraneous": true, - "license": "ISC", - "dependencies": { - "@aws-sdk/client-s3": "3.427.0", - "@aws-sdk/credential-providers": "^3.309.0", - "@aws-sdk/lib-storage": "3.182.0", - "@aws-sdk/s3-request-presigner": "3.173.0", - "@azure/storage-blob": "^12.16.0", - "@google-cloud/storage": "6.5.2", - "@jsonhero/schema-infer": "^0.1.4", - "@project-sunbird/logger": "^0.0.9", - "ajv": "^8.12.0", - "async": "2.6.4", - "aws-sdk": "^2.1348.0", - "axios": "^1.6.0", - "body-parser": "^1.20.2", - "dateformat": "2.0.0", - "express": "^4.18.2", - "http-errors": "^2.0.0", - "http-status": "^1.5.3", - "json-difference": "^1.16.0", - "kafkajs": "^2.2.4", - "knex": "^2.4.2", - "lodash": "4.17.21", - "mime": "^3.0.0", - "moment": "^2.29.4", - "multer": "^1.4.5-lts.1", - "multiparty": "^4.2.1", - "mysql2": "^3.6.3", - "pg": "^8.11.0", - "pg-hstore": "^2.3.4", - "sequelize": "^6.32.0", - "uuid": "3.1.0" - }, - "devDependencies": { - "@babel/core": "^7.18.6", - "@babel/preset-env": "^7.18.6", - "@types/chai": "^4.3.3", - "@types/chai-as-promised": "^7.1.5", - "@types/chai-spies": "^1.0.3", - "@types/express": "^4.17.17", - "@types/http-errors": "^2.0.1", - "@types/lodash": "^4.14.190", - "@types/mime-types": "^2.1.1", - "@types/mocha": "^10.0.1", - "@types/mock-knex": "^0.4.5", - "@types/moment": "^2.13.0", - "@types/multer": "^1.4.7", - "@types/node": "^18.15.11", - "@types/pg": "^8.6.6", - "@types/sinon": "^10.0.15", - "@types/uuid": "^9.0.1", - "babel-loader": "^8.2.5", - "chai": "^4.3.7", - "chai-as-promised": "^7.1.1", - "chai-http": "^4.3.0", - "chai-spies": "^1.0.0", - "diff-json": "^2.0.0", - "fs": "0.0.1-security", - "mocha": "^10.1.0", - "mock-knex": "^0.4.12", - "nock": "^13.2.9", - "nodemon": "^2.0.20", - "nyc": "^15.1.0", - "prettier": "^2.7.1", - "rimraf": "^3.0.2", - "ts-node": "^10.9.1", - "tsconfig-paths": "^4.1.0", - "typescript": "^5.0.4", - "webpack": "^5.73.0", - "webpack-cli": "^5.0.1", - "webpack-dev-server": "^4.9.3" - } - }, - "node_modules/@ampproject/remapping": { - "version": "2.2.0", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@jridgewell/gen-mapping": "^0.1.0", - "@jridgewell/trace-mapping": "^0.3.9" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@aws-crypto/crc32": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz", - "integrity": "sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==", - "dependencies": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/crc32/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/crc32c": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-3.0.0.tgz", - "integrity": "sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==", - "dependencies": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/crc32c/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/ie11-detection": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz", - "integrity": "sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==", - "dependencies": { - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/ie11-detection/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/sha1-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-3.0.0.tgz", - "integrity": "sha512-NJth5c997GLHs6nOYTzFKTbYdMNA6/1XlKVgnZoaZcQ7z7UJlOgj2JdbHE8tiYLS3fzXNCguct77SPGat2raSw==", - "dependencies": { - "@aws-crypto/ie11-detection": "^3.0.0", - "@aws-crypto/supports-web-crypto": "^3.0.0", - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/sha1-browser/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/sha256-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz", - "integrity": "sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==", - "dependencies": { - "@aws-crypto/ie11-detection": "^3.0.0", - "@aws-crypto/sha256-js": "^3.0.0", - "@aws-crypto/supports-web-crypto": "^3.0.0", - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/sha256-browser/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/sha256-js": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz", - "integrity": "sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==", - "dependencies": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/sha256-js/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/supports-web-crypto": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz", - "integrity": "sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==", - "dependencies": { - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/supports-web-crypto/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-crypto/util": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz", - "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==", - "dependencies": { - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - } - }, - "node_modules/@aws-crypto/util/node_modules/tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - }, - "node_modules/@aws-sdk/client-cognito-identity": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.427.0.tgz", - "integrity": "sha512-9brRaNnl6haE7R3R43A5CSNw0k1YtB3xjuArbMg/p6NDUpvRSRgOVNWu2R02Yjh/j2ZuaLOCPLuCipb+PHQPKQ==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.427.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/client-s3": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.590.0.tgz", - "integrity": "sha512-so+pNua0ihsHaSdskw8HCwruoYTAfYSEs3ix4GD1++83C96KaJp3udAutYiCA+84JXg9zitFa7eK7ORJAVZmTw==", - "dependencies": { - "@aws-crypto/sha1-browser": "3.0.0", - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/client-sts": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-bucket-endpoint": "3.587.0", - "@aws-sdk/middleware-expect-continue": "3.577.0", - "@aws-sdk/middleware-flexible-checksums": "3.587.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-location-constraint": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-sdk-s3": "3.587.0", - "@aws-sdk/middleware-signing": "3.587.0", - "@aws-sdk/middleware-ssec": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/signature-v4-multi-region": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@aws-sdk/xml-builder": "3.575.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/eventstream-serde-browser": "^3.0.0", - "@smithy/eventstream-serde-config-resolver": "^3.0.0", - "@smithy/eventstream-serde-node": "^3.0.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-blob-browser": "^3.0.0", - "@smithy/hash-node": "^3.0.0", - "@smithy/hash-stream-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/md5-js": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "@smithy/util-utf8": "^3.0.0", - "@smithy/util-waiter": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/client-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.590.0.tgz", - "integrity": "sha512-6xbC6oQVJKBRTyXyR3C15ksUsPOyW4p+uCj7dlKYWGJvh4vGTV8KhZKS53oPG8t4f1+OMJWjr5wKuXRoaFsmhQ==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/client-sts": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.590.0.tgz", - "integrity": "sha512-f4R1v1LSn4uLYZ5qj4DyL6gp7PXXzJeJsm2seheiJX+53LSF5L7XSDnQVtX1p9Tevv0hp2YUWUTg6QYwIVSuGg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-env": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz", - "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-http": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.587.0.tgz", - "integrity": "sha512-Su1SRWVRCuR1e32oxX3C1V4c5hpPN20WYcRfdcr2wXwHqSvys5DrnmuCC+JoEnS/zt3adUJhPliTqpfKgSdMrA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.590.0.tgz", - "integrity": "sha512-Y5cFciAK38VIvRgZeND7HvFNR32thGtQb8Xop6cMn33FC78uwcRIu9Hc9699XTclCZqz4+Xl1WU+dZ+rnFn2AA==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sts": "^3.590.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-node": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.590.0.tgz", - "integrity": "sha512-Ky38mNFoXobGrDQ11P3dU1e+q1nRJ7eZl8l15KUpvZCe/hOudbxQi/epQrCazD/gRYV2fTyczdLlZzB5ZZ8DhQ==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-ini": "3.590.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-process": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz", - "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.590.0.tgz", - "integrity": "sha512-v+0j/I+je9okfwXsgmLppmwIE+TuMp5WqLz7r7PHz9KjzLyKaKTDvfllFD+8oPpBqnmOWiJ9qTGPkrfhB7a/fQ==", - "dependencies": { - "@aws-sdk/client-sso": "3.590.0", - "@aws-sdk/token-providers": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz", - "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sts": "^3.587.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-host-header": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz", - "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-logger": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz", - "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz", - "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-signing": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.587.0.tgz", - "integrity": "sha512-tiZaTDj4RvhXGRAlncFn7CSEfL3iNPO67WSaxAq+Ls5j1VgczPhu5262cWONNoMgth3nXR1hhLC4ITSl/a6AzA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz", - "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/region-config-resolver": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz", - "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/token-providers": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz", - "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sso-oidc": "^3.587.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/util-endpoints": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz", - "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "@smithy/util-endpoints": "^2.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz", - "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz", - "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/config-resolver": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.1.tgz", - "integrity": "sha512-hbkYJc20SBDz2qqLzttjI/EqXemtmWk0ooRznLsiXp3066KQRTvuKHa7U4jCZCJq6Dozqvy0R1/vNESC9inPJg==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/credential-provider-imds": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz", - "integrity": "sha512-q4A4d38v8pYYmseu/jTS3Z5I3zXlEOe5Obi+EJreVKgSVyWUHOd7/yaVCinC60QG4MRyCs98tcxBH1IMC0bu7Q==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/hash-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.0.tgz", - "integrity": "sha512-84qXstNemP3XS5jcof0el6+bDfjzuvhJPQTEfro3lgtbCtKgzPm3MgiS6ehXVPjeQ5+JS0HqmTz8f/RYfzHVxw==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/invalid-dependency": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.0.tgz", - "integrity": "sha512-F6wBBaEFgJzj0s4KUlliIGPmqXemwP6EavgvDqYwCH40O5Xr2iMHvS8todmGVZtuJCorBkXsYLyTu4PuizVq5g==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-content-length": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.0.tgz", - "integrity": "sha512-3C4s4d/iGobgCtk2tnWW6+zSTOBg1PRAm2vtWZLdriwTroFbbWNSr3lcyzHdrQHnEXYCC5K52EbpfodaIUY8sg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "dependencies": { - "@smithy/types": "^3.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-body-length-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", - "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", - "dependencies": { - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-body-length-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", - "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-defaults-mode-browser": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.3.tgz", - "integrity": "sha512-3DFON2bvXJAukJe+qFgPV/rorG7ZD3m4gjCXHD1V5z/tgKQp5MCTCLntrd686tX6tj8Uli3lefWXJudNg5WmCA==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-defaults-mode-node": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.3.tgz", - "integrity": "sha512-D0b8GJXecT00baoSQ3Iieu3k3mZ7GY8w1zmg8pdogYrGvWJeLcIclqk2gbkG4K0DaBGWrO6v6r20iwIFfDYrmA==", - "dependencies": { - "@smithy/config-resolver": "^3.0.1", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "dependencies": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-s3/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@aws-sdk/client-sso": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.427.0.tgz", - "integrity": "sha512-sFVFEmsQ1rmgYO1SgrOTxE/MTKpeE4hpOkm1WqhLQK7Ij136vXpjCxjH1JYZiHiUzO1wr9t4ex4dlB5J3VS/Xg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.590.0.tgz", - "integrity": "sha512-3yCLPjq6WFfDpdUJKk/gSz4eAPDTjVknXaveMPi2QoVBCshneOnJsV16uNKlpVF1frTHrrDRfKYmbaVh6nFBvQ==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/client-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.590.0.tgz", - "integrity": "sha512-6xbC6oQVJKBRTyXyR3C15ksUsPOyW4p+uCj7dlKYWGJvh4vGTV8KhZKS53oPG8t4f1+OMJWjr5wKuXRoaFsmhQ==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/client-sts": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.590.0.tgz", - "integrity": "sha512-f4R1v1LSn4uLYZ5qj4DyL6gp7PXXzJeJsm2seheiJX+53LSF5L7XSDnQVtX1p9Tevv0hp2YUWUTg6QYwIVSuGg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-env": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz", - "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-http": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.587.0.tgz", - "integrity": "sha512-Su1SRWVRCuR1e32oxX3C1V4c5hpPN20WYcRfdcr2wXwHqSvys5DrnmuCC+JoEnS/zt3adUJhPliTqpfKgSdMrA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.590.0.tgz", - "integrity": "sha512-Y5cFciAK38VIvRgZeND7HvFNR32thGtQb8Xop6cMn33FC78uwcRIu9Hc9699XTclCZqz4+Xl1WU+dZ+rnFn2AA==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sts": "^3.590.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-node": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.590.0.tgz", - "integrity": "sha512-Ky38mNFoXobGrDQ11P3dU1e+q1nRJ7eZl8l15KUpvZCe/hOudbxQi/epQrCazD/gRYV2fTyczdLlZzB5ZZ8DhQ==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-ini": "3.590.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-process": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz", - "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.590.0.tgz", - "integrity": "sha512-v+0j/I+je9okfwXsgmLppmwIE+TuMp5WqLz7r7PHz9KjzLyKaKTDvfllFD+8oPpBqnmOWiJ9qTGPkrfhB7a/fQ==", - "dependencies": { - "@aws-sdk/client-sso": "3.590.0", - "@aws-sdk/token-providers": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz", - "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sts": "^3.587.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/middleware-host-header": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz", - "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/middleware-logger": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz", - "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz", - "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz", - "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/region-config-resolver": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz", - "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/token-providers": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz", - "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-sso-oidc": "^3.587.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/util-endpoints": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz", - "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "@smithy/util-endpoints": "^2.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz", - "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz", - "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/config-resolver": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.1.tgz", - "integrity": "sha512-hbkYJc20SBDz2qqLzttjI/EqXemtmWk0ooRznLsiXp3066KQRTvuKHa7U4jCZCJq6Dozqvy0R1/vNESC9inPJg==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/credential-provider-imds": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz", - "integrity": "sha512-q4A4d38v8pYYmseu/jTS3Z5I3zXlEOe5Obi+EJreVKgSVyWUHOd7/yaVCinC60QG4MRyCs98tcxBH1IMC0bu7Q==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/hash-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.0.tgz", - "integrity": "sha512-84qXstNemP3XS5jcof0el6+bDfjzuvhJPQTEfro3lgtbCtKgzPm3MgiS6ehXVPjeQ5+JS0HqmTz8f/RYfzHVxw==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/invalid-dependency": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.0.tgz", - "integrity": "sha512-F6wBBaEFgJzj0s4KUlliIGPmqXemwP6EavgvDqYwCH40O5Xr2iMHvS8todmGVZtuJCorBkXsYLyTu4PuizVq5g==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-content-length": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.0.tgz", - "integrity": "sha512-3C4s4d/iGobgCtk2tnWW6+zSTOBg1PRAm2vtWZLdriwTroFbbWNSr3lcyzHdrQHnEXYCC5K52EbpfodaIUY8sg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "dependencies": { - "@smithy/types": "^3.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-body-length-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", - "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", - "dependencies": { - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-body-length-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", - "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-defaults-mode-browser": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.3.tgz", - "integrity": "sha512-3DFON2bvXJAukJe+qFgPV/rorG7ZD3m4gjCXHD1V5z/tgKQp5MCTCLntrd686tX6tj8Uli3lefWXJudNg5WmCA==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-defaults-mode-node": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.3.tgz", - "integrity": "sha512-D0b8GJXecT00baoSQ3Iieu3k3mZ7GY8w1zmg8pdogYrGvWJeLcIclqk2gbkG4K0DaBGWrO6v6r20iwIFfDYrmA==", - "dependencies": { - "@smithy/config-resolver": "^3.0.1", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "dependencies": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/client-sso-oidc/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@aws-sdk/client-sts": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.427.0.tgz", - "integrity": "sha512-le2wLJKILyWuRfPz2HbyaNtu5kEki+ojUkTqCU6FPDRrqUvEkaaCBH9Awo/2AtrCfRkiobop8RuTTj6cAnpiJg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-sdk-sts": "3.425.0", - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "fast-xml-parser": "4.2.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/core": { - "version": "3.588.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.588.0.tgz", - "integrity": "sha512-O1c2+9ce46Z+iiid+W3iC1IvPbfIo5ev9CBi54GdNB9SaI8/3+f8MJcux0D6c9toCF0ArMersN/gp8ek57e9uQ==", - "dependencies": { - "@smithy/core": "^2.1.1", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "fast-xml-parser": "4.2.5", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/core/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-cognito-identity": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.427.0.tgz", - "integrity": "sha512-BQNzNrMJlBAfXhYNdAUqaVASpT9Aho5swj7glZKxx4Uds1w5Pih2e14JWgnl8XgUWAZ36pchTrV1aA4JT7N8vw==", - "dependencies": { - "@aws-sdk/client-cognito-identity": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.425.0.tgz", - "integrity": "sha512-J20etnLvMKXRVi5FK4F8yOCNm2RTaQn5psQTGdDEPWJNGxohcSpzzls8U2KcMyUJ+vItlrThr4qwgpHG3i/N0w==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.425.0.tgz", - "integrity": "sha512-aP9nkoVWf+OlNMecrUqe4+RuQrX13nucVbty0HTvuwfwJJj0T6ByWZzle+fo1D+5OxvJmtzTflBWt6jUERdHWA==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.427.0.tgz", - "integrity": "sha512-NmH1cO/w98CKMltYec3IrJIIco19wRjATFNiw83c+FGXZ+InJwReqBnruxIOmKTx2KDzd6fwU1HOewS7UjaaaQ==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.427.0.tgz", - "integrity": "sha512-wYYbQ57nKL8OfgRbl8k6uXcdnYml+p3LSSfDUAuUEp1HKlQ8lOXFJ3BdLr5qrk7LhpyppSRnWBmh2c3kWa7ANQ==", - "dependencies": { - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-ini": "3.427.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.425.0.tgz", - "integrity": "sha512-YY6tkLdvtb1Fgofp3b1UWO+5vwS14LJ/smGmuGpSba0V7gFJRdcrJ9bcb9vVgAGuMdjzRJ+bUKlLLtqXkaykEw==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.427.0.tgz", - "integrity": "sha512-c+tXyS/i49erHs4bAp6vKNYeYlyQ0VNMBgoco0LCn1rL0REtHbfhWMnqDLF6c2n3yIWDOTrQu0D73Idnpy16eA==", - "dependencies": { - "@aws-sdk/client-sso": "3.427.0", - "@aws-sdk/token-providers": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.425.0.tgz", - "integrity": "sha512-/0R65TgRzL01JU3SzloivWNwdkbIhr06uY/F5pBHf/DynQqaspKNfdHn6AiozgSVDfwRHFjKBTUy6wvf3QFkuA==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/credential-providers": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.427.0.tgz", - "integrity": "sha512-rKKohSHju462vo+uQnPjcEZPBAfAMgGH6K1XyyCNpuOC0yYLkG87PYpvAQeb8riTrkHPX0dYUHuTHZ6zQgMGjA==", - "dependencies": { - "@aws-sdk/client-cognito-identity": "3.427.0", - "@aws-sdk/client-sso": "3.427.0", - "@aws-sdk/client-sts": "3.427.0", - "@aws-sdk/credential-provider-cognito-identity": "3.427.0", - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-http": "3.425.0", - "@aws-sdk/credential-provider-ini": "3.427.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/lib-storage": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.427.0.tgz", - "integrity": "sha512-JE26Zo4SMMY2SGlD/FilF5MpLuP8D2IrW+ye/J77dwh91gyGnNa/lubJ7WbVjIAxgeaDHsAkcpNO4VR5k6JCKg==", - "dependencies": { - "@smithy/abort-controller": "^2.0.1", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/smithy-client": "^2.1.9", - "buffer": "5.6.0", - "events": "3.3.0", - "stream-browserify": "3.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-s3": "^3.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.587.0.tgz", - "integrity": "sha512-HkFXLPl8pr6BH/Q0JpOESqEKL0ZK3sk7aSZ1S6GE4RXET7H5R94THULXqQFZzD48gZcyFooO/yNKZTqrZFaWKg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-arn-parser": "3.568.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-bucket-endpoint/node_modules/@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.577.0.tgz", - "integrity": "sha512-6dPp8Tv4F0of4un5IAyG6q++GrRrNQQ4P2NAMB1W0VO4JoEu1C8GievbbDLi88TFIFmtKpnHB0ODCzwnoe8JsA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-expect-continue/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.587.0.tgz", - "integrity": "sha512-URMwp/budDvKhIvZ4a6zIBfFTun/iDlPWXqsGKYjEtHt8jz27OSjCZtDtIeqW4WTBdKL8KZgQcl+DdaE5M1qiQ==", - "dependencies": { - "@aws-crypto/crc32": "3.0.0", - "@aws-crypto/crc32c": "3.0.0", - "@aws-sdk/types": "3.577.0", - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-flexible-checksums/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-host-header": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.425.0.tgz", - "integrity": "sha512-E5Gt41LObQ+cr8QnLthwsH3MtVSNXy1AKJMowDr85h0vzqA/FHUkgHyOGntgozzjXT5M0MaSRYxS0xwTR5D4Ew==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-location-constraint": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.577.0.tgz", - "integrity": "sha512-DKPTD2D2s+t2QUo/IXYtVa/6Un8GZ+phSTBkyBNx2kfZz4Kwavhl/JJzSqTV3GfCXkVdFu7CrjoX7BZ6qWeTUA==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-location-constraint/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-location-constraint/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-logger": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.425.0.tgz", - "integrity": "sha512-INE9XWRXx2f4a/r2vOU0tAmgctVp7nEaEasemNtVBYhqbKLZvr9ndLBSgKGgJ8LIcXAoISipaMuFiqIGkFsm7A==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-recursion-detection": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.425.0.tgz", - "integrity": "sha512-77gnzJ5b91bgD75L/ugpOyerx6lR3oyS4080X1YI58EzdyBMkDrHM4FbMcY2RynETi3lwXCFzLRyZjWXY1mRlw==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.587.0.tgz", - "integrity": "sha512-vtXTGEiw1E9Fax4LmcU2Z208gbrC8ShrdsSLmGcRPpu5NPOGBFBSDG5sy5EDNClrFxIl/Le8coQnD0EDBtx+uQ==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-arn-parser": "3.568.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-s3/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-sdk-sts": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.425.0.tgz", - "integrity": "sha512-JFojrg76oKAoBknnr9EL5N2aJ1mRCtBqXoZYST58GSx8uYdFQ89qS65VNQ8JviBXzsrCNAn4vDhZ5Ch5E6TxGQ==", - "dependencies": { - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-signing": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.425.0.tgz", - "integrity": "sha512-ZpOfgJHk7ovQ0sSwg3tU4NxFOnz53lJlkJRf7S+wxQALHM0P2MJ6LYBrZaFMVsKiJxNIdZBXD6jclgHg72ZW6Q==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/signature-v4": "^2.0.0", - "@smithy/types": "^2.3.4", - "@smithy/util-middleware": "^2.0.3", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/middleware-ssec": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.577.0.tgz", - "integrity": "sha512-i2BPJR+rp8xmRVIGc0h1kDRFcM2J9GnClqqpc+NLSjmYadlcg4mPklisz9HzwFVcRPJ5XcGf3U4BYs5G8+iTyg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-ssec/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-ssec/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/middleware-user-agent": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.427.0.tgz", - "integrity": "sha512-y9HxYsNvnA3KqDl8w1jHeCwz4P9CuBEtu/G+KYffLeAMBsMZmh4SIkFFCO9wE/dyYg6+yo07rYcnnIfy7WA0bw==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/region-config-resolver": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.425.0.tgz", - "integrity": "sha512-u7uv/iUOapIJdRgRkO3wnpYsUgV6ponsZJQgVg/8L+n+Vo5PQL5gAcIuAOwcYSKQPFaeK+KbmByI4SyOK203Vw==", - "dependencies": { - "@smithy/node-config-provider": "^2.0.13", - "@smithy/types": "^2.3.4", - "@smithy/util-config-provider": "^2.0.0", - "@smithy/util-middleware": "^2.0.3", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.590.0.tgz", - "integrity": "sha512-bb8NEG2IUHqFQJsLzr1nlkTZYyokeo3bGbHwMBKZHbdF+OXrQx0kQUcaDCXYWmeydSfHXxweQEJ2U5i1YEvT/A==", - "dependencies": { - "@aws-sdk/signature-v4-multi-region": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-format-url": "3.577.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/s3-request-presigner/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.587.0.tgz", - "integrity": "sha512-TR9+ZSjdXvXUz54ayHcCihhcvxI9W7102J1OK6MrLgBlPE7uRhAx42BR9L5lLJ86Xj3LuqPWf//o9d/zR9WVIg==", - "dependencies": { - "@aws-sdk/middleware-sdk-s3": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/token-providers": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.427.0.tgz", - "integrity": "sha512-4E5E+4p8lJ69PBY400dJXF06LUHYx5lkKzBEsYqWWhoZcoftrvi24ltIhUDoGVLkrLcTHZIWSdFAWSos4hXqeg==", - "dependencies": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/types": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.425.0.tgz", - "integrity": "sha512-6lqbmorwerN4v+J5dqbHPAsjynI0mkEF+blf+69QTaKKGaxBBVaXgqoqul9RXYcK5MMrrYRbQIMd0zYOoy90kA==", - "dependencies": { - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/util-arn-parser": { - "version": "3.568.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.568.0.tgz", - "integrity": "sha512-XUKJWWo+KOB7fbnPP0+g/o5Ulku/X53t7i/h+sPHr5xxYTJJ9CYnbToo95mzxe7xWvkLrsNtJ8L+MnNn9INs2w==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-endpoints": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.427.0.tgz", - "integrity": "sha512-rSyiAIFF/EVvity/+LWUqoTMJ0a25RAc9iqx0WZ4tf1UjuEXRRXxZEb+jEZg1bk+pY84gdLdx9z5E+MSJCZxNQ==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/node-config-provider": "^2.0.13", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.577.0.tgz", - "integrity": "sha512-SyEGC2J+y/krFRuPgiF02FmMYhqbiIkOjDE6k4nYLJQRyS6XEAGxZoG+OHeOVEM+bsDgbxokXZiM3XKGu6qFIg==", - "dependencies": { - "@aws-sdk/types": "3.577.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-format-url/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/util-locate-window": { - "version": "3.310.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.310.0.tgz", - "integrity": "sha512-qo2t/vBTnoXpjKxlsC2e1gBrRm80M3bId27r0BRB2VniSSe7bL1mmzM+/HFtujm0iAxtPM+aLEflLJlJeDPg0w==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@aws-sdk/util-user-agent-browser": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.425.0.tgz", - "integrity": "sha512-22Y9iMtjGcFjGILR6/xdp1qRezlHVLyXtnpEsbuPTiernRCPk6zfAnK/ATH77r02MUjU057tdxVkd5umUBTn9Q==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "bowser": "^2.11.0", - "tslib": "^2.5.0" - } - }, - "node_modules/@aws-sdk/util-user-agent-node": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.425.0.tgz", - "integrity": "sha512-SIR4F5uQeeVAi8lv4OgRirtdtNi5zeyogTuQgGi9su8F/WP1N6JqxofcwpUY5f8/oJ2UlXr/tx1f09UHfJJzvA==", - "dependencies": { - "@aws-sdk/types": "3.425.0", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - }, - "peerDependencies": { - "aws-crt": ">=1.0.0" - }, - "peerDependenciesMeta": { - "aws-crt": { - "optional": true - } - } - }, - "node_modules/@aws-sdk/util-utf8-browser": { - "version": "3.259.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", - "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", - "dependencies": { - "tslib": "^2.3.1" - } - }, - "node_modules/@aws-sdk/xml-builder": { - "version": "3.575.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.575.0.tgz", - "integrity": "sha512-cWgAwmbFYNCFzPwxL705+lWps0F3ZvOckufd2KKoEZUmtpVw9/txUXNrPySUXSmRTSRhoatIMABNfStWR043bQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@aws-sdk/xml-builder/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@azure/abort-controller": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", - "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", - "dependencies": { - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=12.0.0" - } - }, - "node_modules/@azure/core-auth": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.7.2.tgz", - "integrity": "sha512-Igm/S3fDYmnMq1uKS38Ae1/m37B3zigdlZw+kocwEhh5GjyKjPrXKO2J6rzpC1wAxrNil/jX9BJRqBshyjnF3g==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-util": "^1.1.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-auth/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-client": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.2.tgz", - "integrity": "sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-rest-pipeline": "^1.9.1", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.6.1", - "@azure/logger": "^1.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-client/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-http-compat": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.1.2.tgz", - "integrity": "sha512-5MnV1yqzZwgNLLjlizsU3QqOeQChkIXw781Fwh1xdAqJR5AA32IUaq6xv1BICJvfbHoa+JYcaij2HFkhLbNTJQ==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-client": "^1.3.0", - "@azure/core-rest-pipeline": "^1.3.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-http-compat/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-lro": { - "version": "2.5.4", - "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.5.4.tgz", - "integrity": "sha512-3GJiMVH7/10bulzOKGrrLeG/uCBH/9VtxqaMcB9lIqAeamI/xYQSHJL/KcsLDuH+yTjYpro/u6D/MuRe4dN70Q==", - "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-util": "^1.2.0", - "@azure/logger": "^1.0.0", - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@azure/core-paging": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.5.0.tgz", - "integrity": "sha512-zqWdVIt+2Z+3wqxEOGzR5hXFZ8MGKK52x4vFLw8n58pR6ZfKRx3EXYTxTaYxYHc/PexPUTyimcTWFJbji9Z6Iw==", - "dependencies": { - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@azure/core-rest-pipeline": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.16.0.tgz", - "integrity": "sha512-CeuTvsXxCUmEuxH5g/aceuSl6w2EugvNHKAtKKVdiX915EjJJxAwfzNNWZreNnbxHZ2fi0zaM6wwS23x2JVqSQ==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-tracing": "^1.0.1", - "@azure/core-util": "^1.9.0", - "@azure/logger": "^1.0.0", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-rest-pipeline/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-tracing": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.1.2.tgz", - "integrity": "sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-util": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.9.0.tgz", - "integrity": "sha512-AfalUQ1ZppaKuxPPMsFEUdX6GZPB3d9paR9d/TTL7Ow2De8cJaC7ibi7kWVlFAVPCYo31OcnGymc0R89DX8Oaw==", - "dependencies": { - "@azure/abort-controller": "^2.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-util/node_modules/@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-xml": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.4.2.tgz", - "integrity": "sha512-CW3MZhApe/S4iikbYKE7s83fjDBPIr2kpidX+hlGRwh7N4o1nIpQ/PfJTeioqhfqdMvRtheEl+ft64fyTaLNaA==", - "dependencies": { - "fast-xml-parser": "^4.3.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@azure/core-xml/node_modules/fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - }, - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - ], - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/@azure/logger": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.4.tgz", - "integrity": "sha512-ustrPY8MryhloQj7OWGe+HrYx+aoiOxzbXTtgblbV3xwCqpzUK36phH3XNHQKj3EPonyFUuDTfR3qFhTEAuZEg==", - "dependencies": { - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@azure/storage-blob": { - "version": "12.23.0", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.23.0.tgz", - "integrity": "sha512-c1KJ5R5hqR/HtvmFtTn/Y1BNMq45NUBp0LZH7yF8WFMET+wmESgEr0FVTu/Z5NonmfUjbgJZG5Nh8xHc5RdWGQ==", - "dependencies": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-client": "^1.6.2", - "@azure/core-http-compat": "^2.0.0", - "@azure/core-lro": "^2.2.0", - "@azure/core-paging": "^1.1.1", - "@azure/core-rest-pipeline": "^1.10.1", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.6.1", - "@azure/core-xml": "^1.3.2", - "@azure/logger": "^1.0.0", - "events": "^3.0.0", - "tslib": "^2.2.0" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", - "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", - "dev": true, - "dependencies": { - "@babel/highlight": "^7.22.13", - "chalk": "^2.4.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/code-frame/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/code-frame/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/@babel/code-frame/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/code-frame/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.21.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.21.3", - "dev": true, - "license": "MIT", - "dependencies": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.18.6", - "@babel/generator": "^7.21.3", - "@babel/helper-compilation-targets": "^7.20.7", - "@babel/helper-module-transforms": "^7.21.2", - "@babel/helpers": "^7.21.0", - "@babel/parser": "^7.21.3", - "@babel/template": "^7.20.7", - "@babel/traverse": "^7.21.3", - "@babel/types": "^7.21.3", - "convert-source-map": "^1.7.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.2", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/core/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@babel/core/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/generator": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.3.tgz", - "integrity": "sha512-keeZWAV4LU3tW0qRi19HRpabC/ilM0HRBBzf9/k8FFiG4KVpiv0FIy4hHfLfFQZNhziCTPTmd59zoyv6DNISzg==", - "dev": true, - "dependencies": { - "@babel/types": "^7.23.3", - "@jridgewell/gen-mapping": "^0.3.2", - "@jridgewell/trace-mapping": "^0.3.17", - "jsesc": "^2.5.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/generator/node_modules/@jridgewell/gen-mapping": { - "version": "0.3.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/set-array": "^1.0.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.20.7", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.20.5", - "@babel/helper-validator-option": "^7.18.6", - "browserslist": "^4.21.3", - "lru-cache": "^5.1.1", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-environment-visitor": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", - "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-function-name": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", - "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", - "dev": true, - "dependencies": { - "@babel/template": "^7.22.15", - "@babel/types": "^7.23.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-hoist-variables": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", - "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", - "dev": true, - "dependencies": { - "@babel/types": "^7.22.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.18.6", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.18.6" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.21.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-environment-visitor": "^7.18.9", - "@babel/helper-module-imports": "^7.18.6", - "@babel/helper-simple-access": "^7.20.2", - "@babel/helper-split-export-declaration": "^7.18.6", - "@babel/helper-validator-identifier": "^7.19.1", - "@babel/template": "^7.20.7", - "@babel/traverse": "^7.21.2", - "@babel/types": "^7.21.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-simple-access": { - "version": "7.20.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.20.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-split-export-declaration": { - "version": "7.22.6", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", - "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", - "dev": true, - "dependencies": { - "@babel/types": "^7.22.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", - "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", - "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.21.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.21.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.20.7", - "@babel/traverse": "^7.21.0", - "@babel/types": "^7.21.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", - "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", - "dev": true, - "dependencies": { - "@babel/helper-validator-identifier": "^7.22.20", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/highlight/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/@babel/highlight/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/highlight/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/parser": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.3.tgz", - "integrity": "sha512-uVsWNvlVsIninV2prNz/3lHCb+5CJ+e+IUBfbjToAHODtfGYLfCFuY4AU7TskI+dAKk+njsPiBjq1gKTvZOBaw==", - "dev": true, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/template": { - "version": "7.22.15", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", - "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.22.13", - "@babel/parser": "^7.22.15", - "@babel/types": "^7.22.15" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.23.2", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", - "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.22.13", - "@babel/generator": "^7.23.0", - "@babel/helper-environment-visitor": "^7.22.20", - "@babel/helper-function-name": "^7.23.0", - "@babel/helper-hoist-variables": "^7.22.5", - "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/parser": "^7.23.0", - "@babel/types": "^7.23.0", - "debug": "^4.1.0", - "globals": "^11.1.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@babel/traverse/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/types": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.3.tgz", - "integrity": "sha512-OZnvoH2l8PK5eUvEcUyCt/sXgr/h+UWpVuBbOljwcrAgUl6lpchoQ++PHGyQy1AtYnVA6CEq3y5xeEI10brpXw==", - "dev": true, - "dependencies": { - "@babel/helper-string-parser": "^7.22.5", - "@babel/helper-validator-identifier": "^7.22.20", - "to-fast-properties": "^2.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@dabh/diagnostics": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", - "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", - "dependencies": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, - "node_modules/@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", - "dev": true, - "dependencies": { - "eslint-visitor-keys": "^3.3.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "peerDependencies": { - "eslint": "^6.0.0 || ^7.0.0 || >=8.0.0" - } - }, - "node_modules/@eslint-community/regexpp": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.1.tgz", - "integrity": "sha512-Zm2NGpWELsQAD1xsJzGQpYfvICSsFkEpU0jxBjfdC6uNEWXcHnfs9hScFWtXVDVl+rBQJGrl4g1vcKIejpH9dA==", - "dev": true, - "engines": { - "node": "^12.0.0 || ^14.0.0 || >=16.0.0" - } - }, - "node_modules/@eslint/eslintrc": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", - "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", - "dev": true, - "dependencies": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^9.6.0", - "globals": "^13.19.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/@eslint/eslintrc/node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/@eslint/eslintrc/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@eslint/eslintrc/node_modules/globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", - "dev": true, - "dependencies": { - "type-fest": "^0.20.2" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@eslint/eslintrc/node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "node_modules/@eslint/eslintrc/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/@eslint/eslintrc/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@eslint/eslintrc/node_modules/type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", - "dev": true, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, - "node_modules/@google-cloud/paginator": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", - "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", - "dependencies": { - "arrify": "^2.0.0", - "extend": "^3.0.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/projectify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", - "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@google-cloud/promisify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", - "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==", - "engines": { - "node": ">=14" - } - }, - "node_modules/@google-cloud/storage": { - "version": "7.11.1", - "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.11.1.tgz", - "integrity": "sha512-tibLSvgw7nDohMyIelt26kBpJ59YGWA2Rzep++DFNzEzKaSuCSp56Se9iM13ZlM3j5nLzR21IBkpRN58CmvCIw==", - "dependencies": { - "@google-cloud/paginator": "^5.0.0", - "@google-cloud/projectify": "^4.0.0", - "@google-cloud/promisify": "^4.0.0", - "abort-controller": "^3.0.0", - "async-retry": "^1.3.3", - "duplexify": "^4.1.3", - "fast-xml-parser": "^4.3.0", - "gaxios": "^6.0.2", - "google-auth-library": "^9.6.3", - "html-entities": "^2.5.2", - "mime": "^3.0.0", - "p-limit": "^3.0.1", - "retry-request": "^7.0.0", - "teeny-request": "^9.0.0", - "uuid": "^8.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/@google-cloud/storage/node_modules/fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - }, - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - ], - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/@google-cloud/storage/node_modules/mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/@google-cloud/storage/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", - "dev": true, - "dependencies": { - "@humanwhocodes/object-schema": "^2.0.2", - "debug": "^4.3.1", - "minimatch": "^3.0.5" - }, - "engines": { - "node": ">=10.10.0" - } - }, - "node_modules/@humanwhocodes/config-array/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@humanwhocodes/config-array/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/@humanwhocodes/config-array/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "dev": true, - "engines": { - "node": ">=12.22" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/nzakas" - } - }, - "node_modules/@humanwhocodes/object-schema": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", - "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", - "dev": true - }, - "node_modules/@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/argparse": { - "version": "1.0.10", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/camelcase": { - "version": "5.3.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/esprima": { - "version": "4.0.1", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/find-up": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": { - "version": "3.14.1", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/locate-path": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/p-limit": { - "version": "2.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/p-locate": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/load-nyc-config/node_modules/sprintf-js": { - "version": "1.0.3", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.1.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/set-array": "^1.0.0", - "@jridgewell/sourcemap-codec": "^1.4.10" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/set-array": { - "version": "1.1.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.17", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "3.1.0", - "@jridgewell/sourcemap-codec": "1.4.14" - } - }, - "node_modules/@napi-rs/snappy-android-arm-eabi": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.2.2.tgz", - "integrity": "sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-android-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.2.2.tgz", - "integrity": "sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-darwin-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.2.2.tgz", - "integrity": "sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-darwin-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.2.2.tgz", - "integrity": "sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-freebsd-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.2.2.tgz", - "integrity": "sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-arm-gnueabihf": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.2.2.tgz", - "integrity": "sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-arm64-gnu": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.2.2.tgz", - "integrity": "sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-arm64-musl": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.2.2.tgz", - "integrity": "sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-x64-gnu": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.2.2.tgz", - "integrity": "sha512-jZt8Jit/HHDcavt80zxEkDpH+R1Ic0ssiVCoueASzMXa7vwPJeF4ZxZyqUw4qeSy7n8UUExomu8G8ZbP6VKhgw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-linux-x64-musl": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.2.2.tgz", - "integrity": "sha512-Dh96IXgcZrV39a+Tej/owcd9vr5ihiZ3KRix11rr1v0MWtVb61+H1GXXlz6+Zcx9y8jM1NmOuiIuJwkV4vZ4WA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-win32-arm64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.2.2.tgz", - "integrity": "sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-win32-ia32-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.2.2.tgz", - "integrity": "sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@napi-rs/snappy-win32-x64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.2.2.tgz", - "integrity": "sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "dependencies": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "dependencies": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/@project-sunbird/logger": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/@project-sunbird/logger/-/logger-0.0.9.tgz", - "integrity": "sha512-5DLFvgPBa5ffeavwY7fsXQsmZ+CM9rNC3Qmbh9pa4Ygfm10fhucijG0Aqku6WN9i5EeZXLoIOhDGr2fYjKsUqg==", - "dependencies": { - "@types/lodash": "^4.14.149", - "@types/node": "^13.9.0", - "json-stringify-safe": "^5.0.1", - "lodash": "^4.17.15", - "winston": "^3.2.1", - "winston-daily-rotate-file": "^4.4.2" - } - }, - "node_modules/@project-sunbird/logger/node_modules/@types/node": { - "version": "13.13.52", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.52.tgz", - "integrity": "sha512-s3nugnZumCC//n4moGGe6tkNMyYEdaDBitVjwPxXmR5lnMG5dHePinH2EdxkG3Rh1ghFHHixAG4NJhpJW1rthQ==" - }, - "node_modules/@project-sunbird/logger/node_modules/async": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", - "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" - }, - "node_modules/@project-sunbird/logger/node_modules/fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" - }, - "node_modules/@project-sunbird/logger/node_modules/file-stream-rotator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.6.1.tgz", - "integrity": "sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ==", - "dependencies": { - "moment": "^2.29.1" - } - }, - "node_modules/@project-sunbird/logger/node_modules/logform": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.0.tgz", - "integrity": "sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==", - "dependencies": { - "@colors/colors": "1.6.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/@project-sunbird/logger/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "node_modules/@project-sunbird/logger/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/@project-sunbird/logger/node_modules/winston": { - "version": "3.13.0", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.13.0.tgz", - "integrity": "sha512-rwidmA1w3SE4j0E5MuIufFhyJPBDG7Nu71RkZor1p2+qHvJSZ9GYDA81AyleQcZbh/+V6HjeBdfnTZJm9rSeQQ==", - "dependencies": { - "@colors/colors": "^1.6.0", - "@dabh/diagnostics": "^2.0.2", - "async": "^3.2.3", - "is-stream": "^2.0.0", - "logform": "^2.4.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "safe-stable-stringify": "^2.3.1", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.7.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/@project-sunbird/logger/node_modules/winston-daily-rotate-file": { - "version": "4.7.1", - "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-4.7.1.tgz", - "integrity": "sha512-7LGPiYGBPNyGHLn9z33i96zx/bd71pjBn9tqQzO3I4Tayv94WPmBNwKC7CO1wPHdP9uvu+Md/1nr6VSH9h0iaA==", - "dependencies": { - "file-stream-rotator": "^0.6.1", - "object-hash": "^2.0.1", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - }, - "engines": { - "node": ">=8" - }, - "peerDependencies": { - "winston": "^3" - } - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/fake-timers": { - "version": "11.2.2", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz", - "integrity": "sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^3.0.0" - } - }, - "node_modules/@sinonjs/samsam": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz", - "integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^2.0.0", - "lodash.get": "^4.4.2", - "type-detect": "^4.0.8" - } - }, - "node_modules/@sinonjs/samsam/node_modules/@sinonjs/commons": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", - "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", - "dev": true, - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/text-encoding": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz", - "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", - "dev": true - }, - "node_modules/@smithy/abort-controller": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-2.0.11.tgz", - "integrity": "sha512-MSzE1qR2JNyb7ot3blIOT3O3H0Jn06iNDEgHRaqZUwBgx5EG+VIx24Y21tlKofzYryIOcWpIohLrIIyocD6LMA==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-3.0.0.tgz", - "integrity": "sha512-sbnURCwjF0gSToGlsBiAmd1lRCmSn72nu9axfJu5lIx6RUEgHu6GwTMbqCdhQSi0Pumcm5vFxsi9XWXb2mTaoA==", - "dependencies": { - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/chunked-blob-reader-native": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-3.0.0.tgz", - "integrity": "sha512-VDkpCYW+peSuM4zJip5WDfqvg2Mo/e8yxOv3VF1m11y7B8KKMKVFtmZWDe36Fvk8rGuWrPZHHXZ7rR7uM5yWyg==", - "dependencies": { - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/chunked-blob-reader-native/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/config-resolver": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-2.0.14.tgz", - "integrity": "sha512-K1K+FuWQoy8j/G7lAmK85o03O89s2Vvh6kMFmzEmiHUoQCRH1rzbDtMnGNiaMHeSeYJ6y79IyTusdRG+LuWwtg==", - "dependencies": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/types": "^2.3.5", - "@smithy/util-config-provider": "^2.0.0", - "@smithy/util-middleware": "^2.0.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.2.0.tgz", - "integrity": "sha512-ygLZSSKgt9bR8HAxR9mK+U5obvAJBr6zlQuhN5soYWx/amjDoQN4dTkydTypgKe6rIbUjTILyLU+W5XFwXr4kg==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "dependencies": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "dependencies": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "dependencies": { - "@smithy/types": "^3.0.0" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "dependencies": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "dependencies": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "dependencies": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "dependencies": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/core/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@smithy/credential-provider-imds": { - "version": "2.0.16", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-2.0.16.tgz", - "integrity": "sha512-tKa2xF+69TvGxJT+lnJpGrKxUuAZDLYXFhqnPEgnHz+psTpkpcB4QRjHj63+uj83KaeFJdTfW201eLZeRn6FfA==", - "dependencies": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/property-provider": "^2.0.12", - "@smithy/types": "^2.3.5", - "@smithy/url-parser": "^2.0.11", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/eventstream-codec": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-2.0.11.tgz", - "integrity": "sha512-BQCTjxhCYRZIfXapa2LmZSaH8QUBGwMZw7XRN83hrdixbLjIcj+o549zjkedFS07Ve2TlvWUI6BTzP+nv7snBA==", - "dependencies": { - "@aws-crypto/crc32": "3.0.0", - "@smithy/types": "^2.3.5", - "@smithy/util-hex-encoding": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/eventstream-serde-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.0.tgz", - "integrity": "sha512-NB7AFiPN4NxP/YCAnrvYR18z2/ZsiHiF7VtG30gshO9GbFrIb1rC8ep4NGpJSWrz6P64uhPXeo4M0UsCLnZKqw==", - "dependencies": { - "@smithy/eventstream-serde-universal": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-browser/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-config-resolver": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.0.tgz", - "integrity": "sha512-RUQG3vQ3LX7peqqHAbmayhgrF5aTilPnazinaSGF1P0+tgM3vvIRWPHmlLIz2qFqB9LqFIxditxc8O2Z6psrRw==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-config-resolver/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.0.tgz", - "integrity": "sha512-baRPdMBDMBExZXIUAoPGm/hntixjt/VFpU6+VmCyiYJYzRHRxoaI1MN+5XE+hIS8AJ2GCHLMFEIOLzq9xx1EgQ==", - "dependencies": { - "@smithy/eventstream-serde-universal": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-node/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-universal": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.0.tgz", - "integrity": "sha512-HNFfShmotWGeAoW4ujP8meV9BZavcpmerDbPIjkJbxKbN8RsUcpRQ/2OyIxWNxXNH2GWCAxuSB7ynmIGJlQ3Dw==", - "dependencies": { - "@smithy/eventstream-codec": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-universal/node_modules/@smithy/eventstream-codec": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.0.0.tgz", - "integrity": "sha512-PUtyEA0Oik50SaEFCZ0WPVtF9tz/teze2fDptW6WRXl+RrEenH8UbEjudOz8iakiMl3lE3lCVqYf2Y+znL8QFQ==", - "dependencies": { - "@aws-crypto/crc32": "3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/eventstream-serde-universal/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/eventstream-serde-universal/node_modules/@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/fetch-http-handler": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-2.2.2.tgz", - "integrity": "sha512-K7aRtRuaBjzlk+jWWeyfDTLAmRRvmA4fU8eHUXtjsuEDgi3f356ZE32VD2ssxIH13RCLVZbXMt5h7wHzYiSuVA==", - "dependencies": { - "@smithy/protocol-http": "^3.0.7", - "@smithy/querystring-builder": "^2.0.11", - "@smithy/types": "^2.3.5", - "@smithy/util-base64": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/hash-blob-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.0.0.tgz", - "integrity": "sha512-/Wbpdg+bwJvW7lxR/zpWAc1/x/YkcqguuF2bAzkJrvXriZu1vm8r+PUdE4syiVwQg7PPR2dXpi3CLBb9qRDaVQ==", - "dependencies": { - "@smithy/chunked-blob-reader": "^3.0.0", - "@smithy/chunked-blob-reader-native": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/hash-blob-browser/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-node": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-2.0.11.tgz", - "integrity": "sha512-PbleVugN2tbhl1ZoNWVrZ1oTFFas/Hq+s6zGO8B9bv4w/StTriTKA9W+xZJACOj9X7zwfoTLbscM+avCB1KqOQ==", - "dependencies": { - "@smithy/types": "^2.3.5", - "@smithy/util-buffer-from": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/hash-stream-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.0.0.tgz", - "integrity": "sha512-J0i7de+EgXDEGITD4fxzmMX8CyCNETTIRXlxjMiNUvvu76Xn3GJ31wQR85ynlPk2wI1lqoknAFJaD1fiNDlbIA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-stream-node/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-stream-node/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-stream-node/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/hash-stream-node/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/invalid-dependency": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-2.0.11.tgz", - "integrity": "sha512-zazq99ujxYv/NOf9zh7xXbNgzoVLsqE0wle8P/1zU/XdhPi/0zohTPKWUzIxjGdqb5hkkwfBkNkl5H+LE0mvgw==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/is-array-buffer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.0.0.tgz", - "integrity": "sha512-z3PjFjMyZNI98JFRJi/U0nGoLWMSJlDjAW4QUX2WNZLas5C0CmVV6LJ01JI0k90l7FvpmixjWxPFmENSClQ7ug==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/md5-js": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.0.tgz", - "integrity": "sha512-Tm0vrrVzjlD+6RCQTx7D3Ls58S3FUH1ZCtU1MIh/qQmaOo1H9lMN2as6CikcEwgattnA9SURSdoJJ27xMcEfMA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "node_modules/@smithy/md5-js/node_modules/@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/md5-js/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/md5-js/node_modules/@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "dependencies": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/md5-js/node_modules/@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "dependencies": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/middleware-content-length": { - "version": "2.0.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-2.0.13.tgz", - "integrity": "sha512-Md2kxWpaec3bXp1oERFPQPBhOXCkGSAF7uc1E+4rkwjgw3/tqAXRtbjbggu67HJdwaif76As8AV6XxbD1HzqTQ==", - "dependencies": { - "@smithy/protocol-http": "^3.0.7", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/middleware-endpoint": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-2.0.11.tgz", - "integrity": "sha512-mCugsvB15up6fqpzUEpMT4CuJmFkEI+KcozA7QMzYguXCaIilyMKsyxgamwmr+o7lo3QdjN0//XLQ9bWFL129g==", - "dependencies": { - "@smithy/middleware-serde": "^2.0.11", - "@smithy/types": "^2.3.5", - "@smithy/url-parser": "^2.0.11", - "@smithy/util-middleware": "^2.0.4", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/middleware-retry": { - "version": "2.0.16", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-2.0.16.tgz", - "integrity": "sha512-Br5+0yoiMS0ugiOAfJxregzMMGIRCbX4PYo1kDHtLgvkA/d++aHbnHB819m5zOIAMPvPE7AThZgcsoK+WOsUTA==", - "dependencies": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/protocol-http": "^3.0.7", - "@smithy/service-error-classification": "^2.0.4", - "@smithy/types": "^2.3.5", - "@smithy/util-middleware": "^2.0.4", - "@smithy/util-retry": "^2.0.4", - "tslib": "^2.5.0", - "uuid": "^8.3.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/middleware-retry/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/@smithy/middleware-serde": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-2.0.11.tgz", - "integrity": "sha512-NuxnjMyf4zQqhwwdh0OTj5RqpnuT6HcH5Xg5GrPijPcKzc2REXVEVK4Yyk8ckj8ez1XSj/bCmJ+oNjmqB02GWA==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/middleware-stack": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-2.0.5.tgz", - "integrity": "sha512-bVQU/rZzBY7CbSxIrDTGZYnBWKtIw+PL/cRc9B7etZk1IKSOe0NvKMJyWllfhfhrTeMF6eleCzOihIQympAvPw==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/node-config-provider": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-2.1.1.tgz", - "integrity": "sha512-1lF6s1YWBi1LBu2O30tD3jyTgMtuvk/Z1twzXM4GPYe4dmZix4nNREPJIPOcfFikNU2o0eTYP80+izx5F2jIJA==", - "dependencies": { - "@smithy/property-provider": "^2.0.12", - "@smithy/shared-ini-file-loader": "^2.2.0", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/node-http-handler": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-2.1.7.tgz", - "integrity": "sha512-PQIKZXlp3awCDn/xNlCSTFE7aYG/5Tx33M05NfQmWYeB5yV1GZZOSz4dXpwiNJYTXb9jPqjl+ueXXkwtEluFFA==", - "dependencies": { - "@smithy/abort-controller": "^2.0.11", - "@smithy/protocol-http": "^3.0.7", - "@smithy/querystring-builder": "^2.0.11", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/property-provider": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-2.0.12.tgz", - "integrity": "sha512-Un/OvvuQ1Kg8WYtoMCicfsFFuHb/TKL3pCA6ZIo/WvNTJTR94RtoRnL7mY4XkkUAoFMyf6KjcQJ76y1FX7S5rw==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/protocol-http": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-3.0.7.tgz", - "integrity": "sha512-HnZW8y+r66ntYueCDbLqKwWcMNWW8o3eVpSrHNluwtBJ/EUWfQHRKSiu6vZZtc6PGfPQWgVfucoCE/C3QufMAA==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/querystring-builder": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-2.0.11.tgz", - "integrity": "sha512-b4kEbVMxpmfv2VWUITn2otckTi7GlMteZQxi+jlwedoATOGEyrCJPfRcYQJjbCi3fZ2QTfh3PcORvB27+j38Yg==", - "dependencies": { - "@smithy/types": "^2.3.5", - "@smithy/util-uri-escape": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/querystring-parser": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-2.0.11.tgz", - "integrity": "sha512-YXe7jhi7s3dQ0Fu9dLoY/gLu6NCyy8tBWJL/v2c9i7/RLpHgKT+uT96/OqZkHizCJ4kr0ZD46tzMjql/o60KLg==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/service-error-classification": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-2.0.4.tgz", - "integrity": "sha512-77506l12I5gxTZqBkx3Wb0RqMG81bMYLaVQ+EqIWFwQDJRs5UFeXogKxSKojCmz1wLUziHZQXm03MBzPQiumQw==", - "dependencies": { - "@smithy/types": "^2.3.5" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/shared-ini-file-loader": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-2.2.0.tgz", - "integrity": "sha512-xFXqs4vAb5BdkzHSRrTapFoaqS4/3m/CGZzdw46fBjYZ0paYuLAoMY60ICCn1FfGirG+PiJ3eWcqJNe4/SkfyA==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/signature-v4": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-2.0.11.tgz", - "integrity": "sha512-EFVU1dT+2s8xi227l1A9O27edT/GNKvyAK6lZnIZ0zhIHq/jSLznvkk15aonGAM1kmhmZBVGpI7Tt0odueZK9A==", - "dependencies": { - "@smithy/eventstream-codec": "^2.0.11", - "@smithy/is-array-buffer": "^2.0.0", - "@smithy/types": "^2.3.5", - "@smithy/util-hex-encoding": "^2.0.0", - "@smithy/util-middleware": "^2.0.4", - "@smithy/util-uri-escape": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/smithy-client": { - "version": "2.1.10", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-2.1.10.tgz", - "integrity": "sha512-2OEmZDiW1Z196QHuQZ5M6cBE8FCSG0H2HADP1G+DY8P3agsvb0YJyfhyKuJbxIQy15tr3eDAK6FOrlbxgKOOew==", - "dependencies": { - "@smithy/middleware-stack": "^2.0.5", - "@smithy/types": "^2.3.5", - "@smithy/util-stream": "^2.0.15", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/types": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.3.5.tgz", - "integrity": "sha512-ehyDt8M9hehyxrLQGoA1BGPou8Js1Ocoh5M0ngDhJMqbFmNK5N6Xhr9/ZExWkyIW8XcGkiMPq3ZUEE0ScrhbuQ==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/url-parser": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.11.tgz", - "integrity": "sha512-h89yXMCCF+S5k9XIoKltMIWTYj+FcEkU/IIFZ6RtE222fskOTL4Iak6ZRG+ehSvZDt8yKEcxqheTDq7JvvtK3g==", - "dependencies": { - "@smithy/querystring-parser": "^2.0.11", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/util-base64": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-2.0.0.tgz", - "integrity": "sha512-Zb1E4xx+m5Lud8bbeYi5FkcMJMnn+1WUnJF3qD7rAdXpaL7UjkFQLdmW5fHadoKbdHpwH9vSR8EyTJFHJs++tA==", - "dependencies": { - "@smithy/util-buffer-from": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-body-length-browser": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-2.0.0.tgz", - "integrity": "sha512-JdDuS4ircJt+FDnaQj88TzZY3+njZ6O+D3uakS32f2VNnDo3vyEuNdBOh/oFd8Df1zSZOuH1HEChk2AOYDezZg==", - "dependencies": { - "tslib": "^2.5.0" - } - }, - "node_modules/@smithy/util-body-length-node": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-2.1.0.tgz", - "integrity": "sha512-/li0/kj/y3fQ3vyzn36NTLGmUwAICb7Jbe/CsWCktW363gh1MOcpEcSO3mJ344Gv2dqz8YJCLQpb6hju/0qOWw==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-buffer-from": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.0.0.tgz", - "integrity": "sha512-/YNnLoHsR+4W4Vf2wL5lGv0ksg8Bmk3GEGxn2vEQt52AQaPSCuaO5PM5VM7lP1K9qHRKHwrPGktqVoAHKWHxzw==", - "dependencies": { - "@smithy/is-array-buffer": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-config-provider": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-2.0.0.tgz", - "integrity": "sha512-xCQ6UapcIWKxXHEU4Mcs2s7LcFQRiU3XEluM2WcCjjBtQkUN71Tb+ydGmJFPxMUrW/GWMgQEEGipLym4XG0jZg==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-defaults-mode-browser": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-2.0.14.tgz", - "integrity": "sha512-NupG7SWUucm3vJrvlpt9jG1XeoPJphjcivgcUUXhDJbUPy4F04LhlTiAhWSzwlCNcF8OJsMvZ/DWbpYD3pselw==", - "dependencies": { - "@smithy/property-provider": "^2.0.12", - "@smithy/smithy-client": "^2.1.10", - "@smithy/types": "^2.3.5", - "bowser": "^2.11.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@smithy/util-defaults-mode-node": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-2.0.18.tgz", - "integrity": "sha512-+3jMom/b/Cdp21tDnY4vKu249Al+G/P0HbRbct7/aSZDlROzv1tksaYukon6UUv7uoHn+/McqnsvqZHLlqvQ0g==", - "dependencies": { - "@smithy/config-resolver": "^2.0.14", - "@smithy/credential-provider-imds": "^2.0.16", - "@smithy/node-config-provider": "^2.1.1", - "@smithy/property-provider": "^2.0.12", - "@smithy/smithy-client": "^2.1.10", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/@smithy/util-endpoints": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.1.tgz", - "integrity": "sha512-ZRT0VCOnKlVohfoABMc8lWeQo/JEFuPWctfNRXgTHbyOVssMOLYFUNWukxxiHRGVAhV+n3c0kPW+zUqckjVPEA==", - "dependencies": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-endpoints/node_modules/@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "dependencies": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-endpoints/node_modules/@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-endpoints/node_modules/@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-endpoints/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-hex-encoding": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-2.0.0.tgz", - "integrity": "sha512-c5xY+NUnFqG6d7HFh1IFfrm3mGl29lC+vF+geHv4ToiuJCBmIfzx6IeHLg+OgRdPFKDXIw6pvi+p3CsscaMcMA==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-middleware": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-2.0.4.tgz", - "integrity": "sha512-Pbu6P4MBwRcjrLgdTR1O4Y3c0sTZn2JdOiJNcgL7EcIStcQodj+6ZTXtbyU/WTEU3MV2NMA10LxFc3AWHZ3+4A==", - "dependencies": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-retry": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-2.0.4.tgz", - "integrity": "sha512-b+n1jBBKc77C1E/zfBe1Zo7S9OXGBiGn55N0apfhZHxPUP/fMH5AhFUUcWaJh7NAnah284M5lGkBKuhnr3yK5w==", - "dependencies": { - "@smithy/service-error-classification": "^2.0.4", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/@smithy/util-stream": { - "version": "2.0.15", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-2.0.15.tgz", - "integrity": "sha512-A/hkYJPH2N5MCWYvky4tTpQihpYAEzqnUfxDyG3L/yMndy/2sLvxnyQal9Opuj1e9FiKSTeMyjnU9xxZGs0mRw==", - "dependencies": { - "@smithy/fetch-http-handler": "^2.2.2", - "@smithy/node-http-handler": "^2.1.7", - "@smithy/types": "^2.3.5", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-buffer-from": "^2.0.0", - "@smithy/util-hex-encoding": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-uri-escape": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-2.0.0.tgz", - "integrity": "sha512-ebkxsqinSdEooQduuk9CbKcI+wheijxEb3utGXkCoYQkJnwTnLbH1JXGimJtUkQwNQbsbuYwG2+aFVyZf5TLaw==", - "dependencies": { - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-utf8": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.0.0.tgz", - "integrity": "sha512-rctU1VkziY84n5OXe3bPNpKR001ZCME2JCaBBFgtiM2hfKbHFudc/BkMuPab8hRbLd0j3vbnBTTZ1igBf0wgiQ==", - "dependencies": { - "@smithy/util-buffer-from": "^2.0.0", - "tslib": "^2.5.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@smithy/util-waiter": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.0.0.tgz", - "integrity": "sha512-+fEXJxGDLCoqRKVSmo0auGxaqbiCo+8oph+4auefYjaNxjOLKSY2MxVQfRzo65PaZv4fr+5lWg+au7vSuJJ/zw==", - "dependencies": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-waiter/node_modules/@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "dependencies": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@smithy/util-waiter/node_modules/@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=16.0.0" - } - }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "engines": { - "node": ">= 10" - } - }, - "node_modules/@tsconfig/node10": { - "version": "1.0.9", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node12": { - "version": "1.0.11", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node14": { - "version": "1.0.3", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node16": { - "version": "1.0.3", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/body-parser": { - "version": "1.19.2", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "node_modules/@types/caseless": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", - "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" - }, - "node_modules/@types/chai": { - "version": "4.3.4", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/chai-as-promised": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.6.tgz", - "integrity": "sha512-cQLhk8fFarRVZAXUQV1xEnZgMoPxqKojBvRkqPCKPQCzEhpbbSKl1Uu75kDng7k5Ln6LQLUmNBjLlFthCgm1NA==", - "dev": true, - "dependencies": { - "@types/chai": "*" - } - }, - "node_modules/@types/chai-spies": { - "version": "1.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/chai": "*" - } - }, - "node_modules/@types/compression": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.3.tgz", - "integrity": "sha512-rKquEGjebqizyHNMOpaE/4FdYR5VQiWFeesqYfvJU0seSEyB4625UGhNOO/qIkH10S3wftiV7oefc8WdLZ/gCQ==", - "dev": true, - "dependencies": { - "@types/express": "*" - } - }, - "node_modules/@types/connect": { - "version": "3.4.35", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/cookiejar": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/debug": { - "version": "4.1.12", - "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", - "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", - "dependencies": { - "@types/ms": "*" - } - }, - "node_modules/@types/express": { - "version": "4.17.17", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "node_modules/@types/express-serve-static-core": { - "version": "4.17.33", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "node_modules/@types/http-errors": { - "version": "2.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/kafkajs": { - "version": "1.9.0", - "dev": true, - "license": "MIT", - "dependencies": { - "kafkajs": "*" - } - }, - "node_modules/@types/knex": { - "version": "0.16.1", - "dev": true, - "license": "MIT", - "dependencies": { - "knex": "*" - } - }, - "node_modules/@types/lodash": { - "version": "4.14.191", - "license": "MIT" - }, - "node_modules/@types/mime": { - "version": "3.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/mocha": { - "version": "10.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/mock-knex": { - "version": "0.4.6", - "resolved": "https://registry.npmjs.org/@types/mock-knex/-/mock-knex-0.4.6.tgz", - "integrity": "sha512-7MHM9v9ZgFpAZh2NfCyYeEJKlYJSf/mVRAQ324AJZGB9fbo8dVa8u9nxibyJzwwMXYlbRXLNbKEfoK2xBJFgUw==", - "dev": true, - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/moment": { - "version": "2.13.0", - "dev": true, - "license": "MIT", - "dependencies": { - "moment": "*" - } - }, - "node_modules/@types/ms": { - "version": "0.7.34", - "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", - "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==" - }, - "node_modules/@types/node": { - "version": "18.15.3", - "license": "MIT" - }, - "node_modules/@types/pegjs": { - "version": "0.10.6", - "resolved": "https://registry.npmjs.org/@types/pegjs/-/pegjs-0.10.6.tgz", - "integrity": "sha512-eLYXDbZWXh2uxf+w8sXS8d6KSoXTswfps6fvCUuVAGN8eRpfe7h9eSRydxiSJvo9Bf+GzifsDOr9TMQlmJdmkw==" - }, - "node_modules/@types/pg": { - "version": "8.6.6", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "node_modules/@types/qs": { - "version": "6.9.7", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/range-parser": { - "version": "1.2.4", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/request": { - "version": "2.48.12", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", - "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", - "dependencies": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - } - }, - "node_modules/@types/request/node_modules/form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/@types/serve-static": { - "version": "1.15.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/mime": "*", - "@types/node": "*" - } - }, - "node_modules/@types/sinon": { - "version": "17.0.3", - "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-17.0.3.tgz", - "integrity": "sha512-j3uovdn8ewky9kRBG19bOwaZbexJu/XjtkHyjvUgt4xfPFz18dcORIMqnYh66Fx3Powhcr85NT5+er3+oViapw==", - "dev": true, - "dependencies": { - "@types/sinonjs__fake-timers": "*" - } - }, - "node_modules/@types/sinonjs__fake-timers": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.5.tgz", - "integrity": "sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==", - "dev": true - }, - "node_modules/@types/slug": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/@types/slug/-/slug-5.0.8.tgz", - "integrity": "sha512-mblTWR1OST257k1gZ3QvqG+ERSr8Ea6dyM1FH6Jtm4jeXi0/r0/95VNctofuiywPxCVQuE8AuFoqmvJ4iVUlXQ==", - "dev": true - }, - "node_modules/@types/superagent": { - "version": "3.8.7", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/cookiejar": "*", - "@types/node": "*" - } - }, - "node_modules/@types/tough-cookie": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", - "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" - }, - "node_modules/@types/triple-beam": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.3.tgz", - "integrity": "sha512-6tOUG+nVHn0cJbVp25JFayS5UE6+xlbcNF9Lo9mU7U0zk3zeUShZied4YEQZjy1JBF043FSkdXw8YkUJuVtB5g==" - }, - "node_modules/@types/uuid": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.5.tgz", - "integrity": "sha512-xfHdwa1FMJ082prjSJpoEI57GZITiQz10r3vEJCHa2khEFQjKy91aWKz6+zybzssCvXUwE1LQWgWVwZ4nYUvHQ==", - "dev": true - }, - "node_modules/@types/validator": { - "version": "13.11.10", - "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.11.10.tgz", - "integrity": "sha512-e2PNXoXLr6Z+dbfx5zSh9TRlXJrELycxiaXznp4S5+D2M3b9bqJEitNHA5923jhnB2zzFiZHa2f0SI1HoIahpg==" - }, - "node_modules/@typescript-eslint/eslint-plugin": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.12.0.tgz", - "integrity": "sha512-7F91fcbuDf/d3S8o21+r3ZncGIke/+eWk0EpO21LXhDfLahriZF9CGj4fbAetEjlaBdjdSm9a6VeXbpbT6Z40Q==", - "dev": true, - "dependencies": { - "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/type-utils": "7.12.0", - "@typescript-eslint/utils": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "graphemer": "^1.4.0", - "ignore": "^5.3.1", - "natural-compare": "^1.4.0", - "ts-api-utils": "^1.3.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "@typescript-eslint/parser": "^7.0.0", - "eslint": "^8.56.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/parser": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.12.0.tgz", - "integrity": "sha512-dm/J2UDY3oV3TKius2OUZIFHsomQmpHtsV0FTh1WO8EKgHLQ1QCADUqscPgTpU+ih1e21FQSRjXckHn3txn6kQ==", - "dev": true, - "dependencies": { - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/typescript-estree": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "debug": "^4.3.4" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.56.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/parser/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/parser/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@typescript-eslint/scope-manager": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.12.0.tgz", - "integrity": "sha512-itF1pTnN6F3unPak+kutH9raIkL3lhH1YRPGgt7QQOh43DQKVJXmWkpb+vpc/TiDHs6RSd9CTbDsc/Y+Ygq7kg==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/type-utils": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.12.0.tgz", - "integrity": "sha512-lib96tyRtMhLxwauDWUp/uW3FMhLA6D0rJ8T7HmH7x23Gk1Gwwu8UZ94NMXBvOELn6flSPiBrCKlehkiXyaqwA==", - "dev": true, - "dependencies": { - "@typescript-eslint/typescript-estree": "7.12.0", - "@typescript-eslint/utils": "7.12.0", - "debug": "^4.3.4", - "ts-api-utils": "^1.3.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.56.0" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/type-utils/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@typescript-eslint/types": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.12.0.tgz", - "integrity": "sha512-o+0Te6eWp2ppKY3mLCU+YA9pVJxhUJE15FV7kxuD9jgwIAa+w/ycGJBMrYDTpVGUM/tgpa9SeMOugSabWFq7bg==", - "dev": true, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@typescript-eslint/typescript-estree": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.12.0.tgz", - "integrity": "sha512-5bwqLsWBULv1h6pn7cMW5dXX/Y2amRqLaKqsASVwbBHMZSnHqE/HN4vT4fE0aFsiwxYvr98kqOWh1a8ZKXalCQ==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "debug": "^4.3.4", - "globby": "^11.1.0", - "is-glob": "^4.0.3", - "minimatch": "^9.0.4", - "semver": "^7.6.0", - "ts-api-utils": "^1.3.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dev": true, - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@typescript-eslint/typescript-estree/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/@typescript-eslint/utils": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.12.0.tgz", - "integrity": "sha512-Y6hhwxwDx41HNpjuYswYp6gDbkiZ8Hin9Bf5aJQn1bpTs3afYY4GX+MPYxma8jtoIV2GRwTM/UJm/2uGCVv+DQ==", - "dev": true, - "dependencies": { - "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/typescript-estree": "7.12.0" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - }, - "peerDependencies": { - "eslint": "^8.56.0" - } - }, - "node_modules/@typescript-eslint/visitor-keys": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.12.0.tgz", - "integrity": "sha512-uZk7DevrQLL3vSnfFl5bj4sL75qC9D6EdjemIdbtkuUmIheWpuiiylSY01JxJE7+zGrOWDZrp1WxOuDntvKrHQ==", - "dev": true, - "dependencies": { - "@typescript-eslint/types": "7.12.0", - "eslint-visitor-keys": "^3.4.3" - }, - "engines": { - "node": "^18.18.0 || >=20.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/typescript-eslint" - } - }, - "node_modules/@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", - "dev": true - }, - "node_modules/abbrev": { - "version": "1.1.1", - "dev": true, - "license": "ISC" - }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, - "node_modules/accepts": { - "version": "1.3.8", - "license": "MIT", - "dependencies": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/acorn": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", - "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", - "dev": true, - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "peerDependencies": { - "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" - } - }, - "node_modules/acorn-walk": { - "version": "8.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/agent-base": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", - "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "dependencies": { - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/agent-base/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/agent-base/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/aggregate-error": { - "version": "3.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/ajv": { - "version": "8.12.0", - "license": "MIT", - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/ajv-formats": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", - "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", - "dependencies": { - "ajv": "^8.0.0" - }, - "peerDependencies": { - "ajv": "^8.0.0" - }, - "peerDependenciesMeta": { - "ajv": { - "optional": true - } - } - }, - "node_modules/ansi-colors": { - "version": "4.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "devOptional": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "dev": true, - "license": "ISC", - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/append-transform": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "default-require-extensions": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "optional": true - }, - "node_modules/archy": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/are-we-there-yet": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz", - "integrity": "sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==", - "optional": true, - "dependencies": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, - "node_modules/arg": { - "version": "4.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/argparse": { - "version": "2.0.1", - "dev": true, - "license": "Python-2.0" - }, - "node_modules/array-flatten": { - "version": "1.1.1", - "license": "MIT" - }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", - "engines": { - "node": ">=8" - } - }, - "node_modules/assertion-error": { - "version": "1.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/async": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", - "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", - "dependencies": { - "lodash": "^4.17.14" - } - }, - "node_modules/async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "dependencies": { - "retry": "0.13.1" - } - }, - "node_modules/asynckit": { - "version": "0.4.0", - "license": "MIT" - }, - "node_modules/available-typed-arrays": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/aws-sdk": { - "version": "2.1472.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1472.0.tgz", - "integrity": "sha512-U7kAHRbvTy753IXKV8Oom/AqlqnsbXG+Kw5gRbKi6VcsZ3hR/EpNMzdRXTWO5U415bnLWGo8WAqIz67PIaaKsw==", - "dependencies": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "util": "^0.12.4", - "uuid": "8.0.0", - "xml2js": "0.5.0" - }, - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/aws-sdk/node_modules/buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, - "node_modules/aws-sdk/node_modules/events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==", - "engines": { - "node": ">=0.4.x" - } - }, - "node_modules/aws-sdk/node_modules/uuid": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", - "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/axios": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz", - "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==", - "dependencies": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "license": "MIT" - }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/big-integer": { - "version": "1.6.52", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", - "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/bignumber.js": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", - "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==", - "engines": { - "node": "*" - } - }, - "node_modules/binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", - "dependencies": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - }, - "engines": { - "node": "*" - } - }, - "node_modules/binary-extensions": { - "version": "2.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "optional": true, - "dependencies": { - "file-uri-to-path": "1.0.0" - } - }, - "node_modules/bintrees": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", - "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" - }, - "node_modules/bl": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", - "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", - "dependencies": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, - "node_modules/body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", - "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/bowser": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", - "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" - }, - "node_modules/brace-expansion": { - "version": "1.1.11", - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/browser-stdout": { - "version": "1.3.1", - "dev": true, - "license": "ISC" - }, - "node_modules/browserslist": { - "version": "4.21.5", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - } - ], - "license": "MIT", - "dependencies": { - "caniuse-lite": "^1.0.30001449", - "electron-to-chromium": "^1.4.284", - "node-releases": "^2.0.8", - "update-browserslist-db": "^1.0.10" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/buffer": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", - "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "dependencies": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, - "node_modules/buffer-alloc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", - "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", - "optional": true, - "dependencies": { - "buffer-alloc-unsafe": "^1.1.0", - "buffer-fill": "^1.0.0" - } - }, - "node_modules/buffer-alloc-unsafe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", - "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", - "optional": true - }, - "node_modules/buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==", - "engines": { - "node": "*" - } - }, - "node_modules/buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" - }, - "node_modules/buffer-fill": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", - "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==", - "optional": true - }, - "node_modules/buffermaker": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/buffermaker/-/buffermaker-1.2.1.tgz", - "integrity": "sha512-IdnyU2jDHU65U63JuVQNTHiWjPRH0CS3aYd/WPaEwyX84rFdukhOduAVb1jwUScmb5X0JWPw8NZOrhoLMiyAHQ==", - "dependencies": { - "long": "1.1.2" - } - }, - "node_modules/buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==", - "engines": { - "node": ">=0.2.0" - } - }, - "node_modules/bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/caching-transform": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "hasha": "^5.0.0", - "make-dir": "^3.0.0", - "package-hash": "^4.0.0", - "write-file-atomic": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/call-bind": { - "version": "1.0.2", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/camelcase": { - "version": "6.3.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001466", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/chai": { - "version": "4.3.7", - "dev": true, - "license": "MIT", - "dependencies": { - "assertion-error": "^1.1.0", - "check-error": "^1.0.2", - "deep-eql": "^4.1.2", - "get-func-name": "^2.0.0", - "loupe": "^2.3.1", - "pathval": "^1.1.1", - "type-detect": "^4.0.5" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/chai-as-promised": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", - "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", - "dev": true, - "dependencies": { - "check-error": "^1.0.2" - }, - "peerDependencies": { - "chai": ">= 2.1.2 < 5" - } - }, - "node_modules/chai-http": { - "version": "4.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/chai": "4", - "@types/superagent": "^3.8.3", - "cookiejar": "^2.1.1", - "is-ip": "^2.0.0", - "methods": "^1.1.2", - "qs": "^6.5.1", - "superagent": "^3.7.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/chai-spies": { - "version": "1.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 4.0.0" - }, - "peerDependencies": { - "chai": "*" - } - }, - "node_modules/chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", - "dependencies": { - "traverse": ">=0.3.0 <0.4" - }, - "engines": { - "node": "*" - } - }, - "node_modules/chalk": { - "version": "4.1.2", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/chalk/node_modules/supports-color": { - "version": "7.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/check-error": { - "version": "1.0.2", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/chokidar": { - "version": "3.5.3", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - ], - "license": "MIT", - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, - "node_modules/chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "optional": true - }, - "node_modules/clean-stack": { - "version": "2.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/cliui": { - "version": "7.0.4", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "node_modules/code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/color": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", - "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", - "dependencies": { - "color-convert": "^1.9.3", - "color-string": "^1.6.0" - } - }, - "node_modules/color-convert": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "license": "MIT" - }, - "node_modules/color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "dependencies": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "node_modules/color/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/color/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" - }, - "node_modules/colorette": { - "version": "2.0.19", - "license": "MIT" - }, - "node_modules/colors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", - "integrity": "sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/colorspace": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", - "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", - "dependencies": { - "color": "^3.1.3", - "text-hex": "1.0.x" - } - }, - "node_modules/combined-stream": { - "version": "1.0.8", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/commander": { - "version": "9.5.0", - "license": "MIT", - "engines": { - "node": "^12.20.0 || >=14" - } - }, - "node_modules/commondir": { - "version": "1.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/component-emitter": { - "version": "1.3.0", - "dev": true, - "license": "MIT" - }, - "node_modules/compressible": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", - "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", - "dependencies": { - "mime-db": ">= 1.43.0 < 2" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", - "dependencies": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", - "debug": "2.6.9", - "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/compression/node_modules/bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/compression/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/concat-map": { - "version": "0.0.1", - "license": "MIT" - }, - "node_modules/console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", - "optional": true - }, - "node_modules/content-disposition": { - "version": "0.5.4", - "license": "MIT", - "dependencies": { - "safe-buffer": "5.2.1" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/content-type": { - "version": "1.0.5", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/convert-source-map": { - "version": "1.9.0", - "dev": true, - "license": "MIT" - }, - "node_modules/cookie": { - "version": "0.5.0", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/cookie-signature": { - "version": "1.0.6", - "license": "MIT" - }, - "node_modules/cookiejar": { - "version": "2.1.4", - "dev": true, - "license": "MIT" - }, - "node_modules/core-util-is": { - "version": "1.0.3", - "license": "MIT" - }, - "node_modules/create-require": { - "version": "1.1.1", - "dev": true, - "license": "MIT" - }, - "node_modules/cross-spawn": { - "version": "7.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/cycle": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", - "integrity": "sha512-TVF6svNzeQCOpjCqsy0/CSy8VgObG3wXusJ73xW2GbG5rGx7lC8zxDSURicsXI2UsGdi2L0QNRCi745/wUDvsA==", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/date-format": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/date-format/-/date-format-4.0.14.tgz", - "integrity": "sha512-39BOQLs9ZjKh0/patS9nrT8wc3ioX3/eA/zgbKNopnF2wCqJEoxywwwElATYvRsXdnOxA/OQeQoFZ3rFjVajhg==", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/dateformat": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.0.0.tgz", - "integrity": "sha512-k6+FtJ8RoNx9V0yHo6lURQWlFqc8wbo0t/kocHvfMJQiXTW+izR6bubmB9HeWuUiZFtpMrAm+wPMGmGhfPbNlQ==", - "engines": { - "node": "*" - } - }, - "node_modules/debug": { - "version": "2.6.9", - "license": "MIT", - "dependencies": { - "ms": "2.0.0" - } - }, - "node_modules/decamelize": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", - "optional": true, - "dependencies": { - "mimic-response": "^1.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/deep-eql": { - "version": "4.1.3", - "dev": true, - "license": "MIT", - "dependencies": { - "type-detect": "^4.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "optional": true, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true - }, - "node_modules/default-require-extensions": { - "version": "3.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "strip-bom": "^4.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", - "optional": true - }, - "node_modules/denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==", - "engines": { - "node": ">=0.10" - } - }, - "node_modules/depd": { - "version": "2.0.0", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/destroy": { - "version": "1.2.0", - "license": "MIT", - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/detect-libc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", - "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", - "optional": true, - "bin": { - "detect-libc": "bin/detect-libc.js" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/diff": { - "version": "5.0.0", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/diff-json": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "lodash.difference": ">= 4.0.0", - "lodash.find": ">= 4.0.0", - "lodash.intersection": ">= 4.0.0", - "lodash.keyby": ">= 4.0.0" - } - }, - "node_modules/dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "dependencies": { - "path-type": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/doctrine": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", - "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", - "dev": true, - "dependencies": { - "esutils": "^2.0.2" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/dottie": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.6.tgz", - "integrity": "sha512-iGCHkfUc5kFekGiqhe8B/mdaurD+lakO9txNnTvKtA6PISrw86LgqHvRzWYPyoE2Ph5aMIrCw9/uko6XHTKCwA==" - }, - "node_modules/duplexify": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", - "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", - "dependencies": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - } - }, - "node_modules/duplexify/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "dependencies": { - "safe-buffer": "^5.0.1" - } - }, - "node_modules/ee-first": { - "version": "1.1.1", - "license": "MIT" - }, - "node_modules/electron-to-chromium": { - "version": "1.4.330", - "dev": true, - "license": "ISC" - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "devOptional": true, - "license": "MIT" - }, - "node_modules/enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "node_modules/encodeurl": { - "version": "1.0.2", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dependencies": { - "once": "^1.4.0" - } - }, - "node_modules/es6-error": { - "version": "4.1.1", - "dev": true, - "license": "MIT" - }, - "node_modules/escalade": { - "version": "3.1.1", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-html": { - "version": "1.0.3", - "license": "MIT" - }, - "node_modules/escape-string-regexp": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", - "dev": true, - "dependencies": { - "@eslint-community/eslint-utils": "^4.2.0", - "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", - "@humanwhocodes/module-importer": "^1.0.1", - "@nodelib/fs.walk": "^1.2.8", - "@ungap/structured-clone": "^1.2.0", - "ajv": "^6.12.4", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.2", - "debug": "^4.3.2", - "doctrine": "^3.0.0", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^7.2.2", - "eslint-visitor-keys": "^3.4.3", - "espree": "^9.6.1", - "esquery": "^1.4.2", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^6.0.1", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "globals": "^13.19.0", - "graphemer": "^1.4.0", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "is-path-inside": "^3.0.3", - "js-yaml": "^4.1.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3", - "strip-ansi": "^6.0.1", - "text-table": "^0.2.0" - }, - "bin": { - "eslint": "bin/eslint.js" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-scope": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", - "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", - "dev": true, - "dependencies": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/eslint/node_modules/ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/eslint/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/eslint/node_modules/glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.3" - }, - "engines": { - "node": ">=10.13.0" - } - }, - "node_modules/eslint/node_modules/globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", - "dev": true, - "dependencies": { - "type-fest": "^0.20.2" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/eslint/node_modules/json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "node_modules/eslint/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/eslint/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "node_modules/eslint/node_modules/type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/esm": { - "version": "3.2.25", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/espree": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", - "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", - "dev": true, - "dependencies": { - "acorn": "^8.9.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.4.1" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - }, - "funding": { - "url": "https://opencollective.com/eslint" - } - }, - "node_modules/esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", - "dev": true, - "dependencies": { - "estraverse": "^5.1.0" - }, - "engines": { - "node": ">=0.10" - } - }, - "node_modules/esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "dependencies": { - "estraverse": "^5.2.0" - }, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, - "node_modules/esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/etag": { - "version": "1.8.1", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "engines": { - "node": ">=6" - } - }, - "node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "engines": { - "node": ">=0.8.x" - } - }, - "node_modules/expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "optional": true, - "engines": { - "node": ">=6" - } - }, - "node_modules/express": { - "version": "4.18.2", - "license": "MIT", - "dependencies": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.1", - "content-disposition": "0.5.4", - "content-type": "~1.0.4", - "cookie": "0.5.0", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "1.2.0", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", - "methods": "~1.1.2", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.7", - "qs": "6.11.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "engines": { - "node": ">= 0.10.0" - } - }, - "node_modules/express/node_modules/body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", - "dependencies": { - "bytes": "3.1.2", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.1", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8", - "npm": "1.2.8000 || >= 1.4.16" - } - }, - "node_modules/express/node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", - "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/extend": { - "version": "3.0.2", - "license": "MIT" - }, - "node_modules/eyes": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", - "integrity": "sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==", - "engines": { - "node": "> 0.1.90" - } - }, - "node_modules/fast-deep-equal": { - "version": "3.1.3", - "license": "MIT" - }, - "node_modules/fast-glob": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "dev": true, - "dependencies": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - }, - "engines": { - "node": ">=8.6.0" - } - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true - }, - "node_modules/fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true - }, - "node_modules/fast-safe-stringify": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", - "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" - }, - "node_modules/fast-xml-parser": { - "version": "4.2.5", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", - "integrity": "sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==", - "funding": [ - { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - }, - { - "type": "github", - "url": "https://github.com/sponsors/NaturalIntelligence" - } - ], - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - } - }, - "node_modules/fastq": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", - "dev": true, - "dependencies": { - "reusify": "^1.0.4" - } - }, - "node_modules/fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "dependencies": { - "pend": "~1.2.0" - } - }, - "node_modules/fecha": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz", - "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg==" - }, - "node_modules/file-entry-cache": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", - "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", - "dev": true, - "dependencies": { - "flat-cache": "^3.0.4" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" - } - }, - "node_modules/file-stream-rotator": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.2.1.tgz", - "integrity": "sha512-Qf6xva6g0fWqI3LR48eOa8ubF+AP6ftLUSt9Uin6XadP5s3d4vD0J52l/JjmMPiFleCtsVJJ6cyMU2GNRLXQIQ==", - "dependencies": { - "moment": "^2.11.2" - } - }, - "node_modules/file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "optional": true - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/finalhandler": { - "version": "1.2.0", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/find-cache-dir": { - "version": "3.3.2", - "dev": true, - "license": "MIT", - "dependencies": { - "commondir": "^1.0.1", - "make-dir": "^3.0.2", - "pkg-dir": "^4.1.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/avajs/find-cache-dir?sponsor=1" - } - }, - "node_modules/find-up": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/flat": { - "version": "5.0.2", - "dev": true, - "license": "BSD-3-Clause", - "bin": { - "flat": "cli.js" - } - }, - "node_modules/flat-cache": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", - "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", - "dev": true, - "dependencies": { - "flatted": "^3.2.9", - "keyv": "^4.5.3", - "rimraf": "^3.0.2" - }, - "engines": { - "node": "^10.12.0 || >=12.0.0" - } - }, - "node_modules/flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==" - }, - "node_modules/fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, - "node_modules/follow-redirects": { - "version": "1.15.2", - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/RubenVerborgh" - } - ], - "license": "MIT", - "engines": { - "node": ">=4.0" - }, - "peerDependenciesMeta": { - "debug": { - "optional": true - } - } - }, - "node_modules/for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dependencies": { - "is-callable": "^1.1.3" - } - }, - "node_modules/foreground-child": { - "version": "2.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^3.0.2" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/form-data": { - "version": "4.0.0", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/formidable": { - "version": "1.2.6", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://ko-fi.com/tunnckoCore/commissions" - } - }, - "node_modules/forwarded": { - "version": "0.2.0", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fresh": { - "version": "0.5.2", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/fromentries": { - "version": "1.3.2", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "optional": true - }, - "node_modules/fs-extra": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", - "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", - "dependencies": { - "graceful-fs": "^4.2.0", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - }, - "engines": { - "node": ">=6 <7 || >=8" - } - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/fsevents": { - "version": "2.3.2", - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.1", - "license": "MIT" - }, - "node_modules/gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha512-14x4kjc6lkD3ltw589k0NrPD6cCNTD6CWoVUNpB85+DrtONoZn+Rug6xZU5RvSC4+TZPxA5AnBibQYAvZn41Hg==", - "optional": true, - "dependencies": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - } - }, - "node_modules/gauge/node_modules/ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/gauge/node_modules/is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==", - "optional": true, - "dependencies": { - "number-is-nan": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/gauge/node_modules/string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==", - "optional": true, - "dependencies": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/gauge/node_modules/strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", - "optional": true, - "dependencies": { - "ansi-regex": "^2.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/gaxios": { - "version": "6.6.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.6.0.tgz", - "integrity": "sha512-bpOZVQV5gthH/jVCSuYuokRo2bTKOcuBiVWpjmTn6C5Agl5zclGfTljuGsQZxwwDBkli+YhZhP4TdlqTnhOezQ==", - "dependencies": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gaxios/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/gcp-metadata": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", - "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", - "dependencies": { - "gaxios": "^6.0.0", - "json-bigint": "^1.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "dev": true, - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-func-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", - "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", - "dev": true, - "engines": { - "node": "*" - } - }, - "node_modules/get-intrinsic": { - "version": "1.2.0", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/get-package-type": { - "version": "0.1.0", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/getopts": { - "version": "2.3.0", - "license": "MIT" - }, - "node_modules/github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "optional": true - }, - "node_modules/glob": { - "version": "7.2.0", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/glob/node_modules/minimatch": { - "version": "3.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/globals": { - "version": "11.12.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "dependencies": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/google-auth-library": { - "version": "9.10.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.10.0.tgz", - "integrity": "sha512-ol+oSa5NbcGdDqA+gZ3G3mev59OHBZksBTxY/tYwjtcp1H/scAFwJfSQU9/1RALoyZ7FslNbke8j4i3ipwlyuQ==", - "dependencies": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dependencies": { - "get-intrinsic": "^1.1.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.10", - "license": "ISC" - }, - "node_modules/graphemer": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", - "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true - }, - "node_modules/gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "dependencies": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/has": { - "version": "1.0.3", - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.1" - }, - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/has-symbols": { - "version": "1.0.3", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "dependencies": { - "has-symbols": "^1.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", - "optional": true - }, - "node_modules/hasha": { - "version": "5.2.2", - "dev": true, - "license": "MIT", - "dependencies": { - "is-stream": "^2.0.0", - "type-fest": "^0.8.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/he": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "bin": { - "he": "bin/he" - } - }, - "node_modules/html-entities": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", - "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/mdevils" - }, - { - "type": "patreon", - "url": "https://patreon.com/mdevils" - } - ] - }, - "node_modules/html-escaper": { - "version": "2.0.2", - "dev": true, - "license": "MIT" - }, - "node_modules/http-errors": { - "version": "2.0.0", - "license": "MIT", - "dependencies": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "dependencies": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/http-proxy-agent/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/http-proxy-agent/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/http-status": { - "version": "1.6.2", - "license": "BSD-3-Clause", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/https-proxy-agent": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", - "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", - "dependencies": { - "agent-base": "^7.0.2", - "debug": "4" - }, - "engines": { - "node": ">= 14" - } - }, - "node_modules/https-proxy-agent/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/https-proxy-agent/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dependencies": { - "safer-buffer": ">= 2.1.2 < 3" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, - "node_modules/ignore": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", - "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", - "dev": true, - "engines": { - "node": ">= 4" - } - }, - "node_modules/ignore-by-default": { - "version": "1.0.1", - "dev": true, - "license": "ISC" - }, - "node_modules/import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", - "dev": true, - "dependencies": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/import-fresh/node_modules/resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/indent-string": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/inflection": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.4.tgz", - "integrity": "sha512-6I/HUDeYFfuNCVS3td055BaXBwKYuzw7K3ExVMStBowKo9oOAMJIXIHvdyR3iboTCp1b+1i5DSkIZTcwIktuDw==", - "engines": [ - "node >= 0.4.0" - ] - }, - "node_modules/inflight": { - "version": "1.0.6", - "dev": true, - "license": "ISC", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "license": "ISC" - }, - "node_modules/ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "optional": true - }, - "node_modules/interpret": { - "version": "2.2.0", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/ip-regex": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/ipaddr.js": { - "version": "1.9.1", - "license": "MIT", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "dependencies": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, - "node_modules/is-binary-path": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "binary-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-core-module": { - "version": "2.11.0", - "license": "MIT", - "dependencies": { - "has": "^1.0.3" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-extglob": { - "version": "2.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "devOptional": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "dependencies": { - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-glob": { - "version": "4.0.3", - "dev": true, - "license": "MIT", - "dependencies": { - "is-extglob": "^2.1.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/is-ip": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ip-regex": "^2.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-path-inside": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", - "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/is-plain-obj": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-typed-array": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", - "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", - "dependencies": { - "which-typed-array": "^1.1.11" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-typedarray": { - "version": "1.0.0", - "dev": true, - "license": "MIT" - }, - "node_modules/is-unicode-supported": { - "version": "0.1.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-windows": { - "version": "1.0.2", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/isarray": { - "version": "1.0.0", - "license": "MIT" - }, - "node_modules/isexe": { - "version": "2.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" - }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.0", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-hook": { - "version": "3.0.0", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "append-transform": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument": { - "version": "4.0.3", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.7.5", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.0.0", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-processinfo": { - "version": "2.0.3", - "dev": true, - "license": "ISC", - "dependencies": { - "archy": "^1.0.0", - "cross-spawn": "^7.0.3", - "istanbul-lib-coverage": "^3.2.0", - "p-map": "^3.0.0", - "rimraf": "^3.0.0", - "uuid": "^8.3.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-processinfo/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true, - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.0", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^3.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-report/node_modules/supports-color": { - "version": "7.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-source-maps": { - "version": "4.0.1", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-source-maps/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/istanbul-lib-source-maps/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/istanbul-reports": { - "version": "3.1.5", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", - "engines": { - "node": ">= 0.6.0" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true - }, - "node_modules/js-yaml": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/jsesc": { - "version": "2.5.2", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "dependencies": { - "bignumber.js": "^9.0.0" - } - }, - "node_modules/json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true - }, - "node_modules/json-schema-traverse": { - "version": "1.0.0", - "license": "MIT" - }, - "node_modules/json-stable-stringify-without-jsonify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true - }, - "node_modules/json-stringify-safe": { - "version": "5.0.1", - "license": "ISC" - }, - "node_modules/json5": { - "version": "2.2.3", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", - "optionalDependencies": { - "graceful-fs": "^4.1.6" - } - }, - "node_modules/just-extend": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", - "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==", - "dev": true - }, - "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "dependencies": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "dependencies": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, - "node_modules/kafka-node": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/kafka-node/-/kafka-node-5.0.0.tgz", - "integrity": "sha512-dD2ga5gLcQhsq1yNoQdy1MU4x4z7YnXM5bcG9SdQuiNr5KKuAmXixH1Mggwdah5o7EfholFbcNDPSVA6BIfaug==", - "dependencies": { - "async": "^2.6.2", - "binary": "~0.3.0", - "bl": "^2.2.0", - "buffer-crc32": "~0.2.5", - "buffermaker": "~1.2.0", - "debug": "^2.1.3", - "denque": "^1.3.0", - "lodash": "^4.17.4", - "minimatch": "^3.0.2", - "nested-error-stacks": "^2.0.0", - "optional": "^0.1.3", - "retry": "^0.10.1", - "uuid": "^3.0.0" - }, - "engines": { - "node": ">=8.5.1" - }, - "optionalDependencies": { - "snappy": "^6.0.1" - } - }, - "node_modules/kafka-node/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/kafka-node/node_modules/retry": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz", - "integrity": "sha512-ZXUSQYTHdl3uS7IuCehYfMzKyIDBNoAuUblvy5oGO5UJSUTmStUUVPXbA9Qxd173Bgre53yCQczQuHgRWAdvJQ==", - "engines": { - "node": "*" - } - }, - "node_modules/kafkajs": { - "version": "2.2.4", - "license": "MIT", - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/kafkajs-snappy": { - "version": "1.1.0", - "license": "MIT", - "dependencies": { - "snappyjs": "^0.6.0" - }, - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/kafkajs-snappy-typescript": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/kafkajs-snappy-typescript/-/kafkajs-snappy-typescript-1.0.3.tgz", - "integrity": "sha512-H5CiRKQ+RGJprWixvY5gDF3ofZEqQA0FFo0ePNgDCoJuH87n3gGUogtlUuei6A8c1ad/5qsQS3f9SSnmB/me4A==", - "dependencies": { - "snappy": "^7.2.2" - } - }, - "node_modules/kafkajs-snappy-typescript/node_modules/snappy": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/snappy/-/snappy-7.2.2.tgz", - "integrity": "sha512-iADMq1kY0v3vJmGTuKcFWSXt15qYUz7wFkArOrsSg0IFfI3nJqIJvK2/ZbEIndg7erIJLtAVX2nSOqPz7DcwbA==", - "engines": { - "node": ">= 10" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "optionalDependencies": { - "@napi-rs/snappy-android-arm-eabi": "7.2.2", - "@napi-rs/snappy-android-arm64": "7.2.2", - "@napi-rs/snappy-darwin-arm64": "7.2.2", - "@napi-rs/snappy-darwin-x64": "7.2.2", - "@napi-rs/snappy-freebsd-x64": "7.2.2", - "@napi-rs/snappy-linux-arm-gnueabihf": "7.2.2", - "@napi-rs/snappy-linux-arm64-gnu": "7.2.2", - "@napi-rs/snappy-linux-arm64-musl": "7.2.2", - "@napi-rs/snappy-linux-x64-gnu": "7.2.2", - "@napi-rs/snappy-linux-x64-musl": "7.2.2", - "@napi-rs/snappy-win32-arm64-msvc": "7.2.2", - "@napi-rs/snappy-win32-ia32-msvc": "7.2.2", - "@napi-rs/snappy-win32-x64-msvc": "7.2.2" - } - }, - "node_modules/keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, - "dependencies": { - "json-buffer": "3.0.1" - } - }, - "node_modules/knex": { - "version": "2.4.2", - "license": "MIT", - "dependencies": { - "colorette": "2.0.19", - "commander": "^9.1.0", - "debug": "4.3.4", - "escalade": "^3.1.1", - "esm": "^3.2.25", - "get-package-type": "^0.1.0", - "getopts": "2.3.0", - "interpret": "^2.2.0", - "lodash": "^4.17.21", - "pg-connection-string": "2.5.0", - "rechoir": "^0.8.0", - "resolve-from": "^5.0.0", - "tarn": "^3.0.2", - "tildify": "2.0.0" - }, - "bin": { - "knex": "bin/cli.js" - }, - "engines": { - "node": ">=12" - }, - "peerDependenciesMeta": { - "better-sqlite3": { - "optional": true - }, - "mysql": { - "optional": true - }, - "mysql2": { - "optional": true - }, - "pg": { - "optional": true - }, - "pg-native": { - "optional": true - }, - "sqlite3": { - "optional": true - }, - "tedious": { - "optional": true - } - } - }, - "node_modules/knex/node_modules/debug": { - "version": "4.3.4", - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/knex/node_modules/ms": { - "version": "2.1.2", - "license": "MIT" - }, - "node_modules/kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, - "node_modules/levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "dependencies": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/locate-path": { - "version": "6.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^5.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/lodash": { - "version": "4.17.21", - "license": "MIT" - }, - "node_modules/lodash.difference": { - "version": "4.5.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.find": { - "version": "4.6.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.flattendeep": { - "version": "4.4.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.get": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", - "dev": true - }, - "node_modules/lodash.intersection": { - "version": "4.4.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.keyby": { - "version": "4.6.0", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true - }, - "node_modules/log-symbols": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/log4js": { - "version": "6.9.1", - "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.9.1.tgz", - "integrity": "sha512-1somDdy9sChrr9/f4UlzhdaGfDR2c/SaD2a4T7qEkG4jTS57/B3qmnjLYePwQ8cqWnUHZI0iAKxMBpCZICiZ2g==", - "dependencies": { - "date-format": "^4.0.14", - "debug": "^4.3.4", - "flatted": "^3.2.7", - "rfdc": "^1.3.0", - "streamroller": "^3.1.5" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/log4js/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/log4js/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/logform": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-1.10.0.tgz", - "integrity": "sha512-em5ojIhU18fIMOw/333mD+ZLE2fis0EzXl1ZwHx4iQzmpQi6odNiY/t+ITNr33JZhT9/KEaH+UPIipr6a9EjWg==", - "dependencies": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^2.3.3", - "ms": "^2.1.1", - "triple-beam": "^1.2.0" - } - }, - "node_modules/logform/node_modules/colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/logform/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "node_modules/long": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/long/-/long-1.1.2.tgz", - "integrity": "sha512-pjR3OP1X2VVQhCQlrq3s8UxugQsuoucwMOn9Yj/kN/61HMc+lDFJS5bvpNEHneZ9NVaSm8gNWxZvtGS7lqHb3Q==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/loupe": { - "version": "2.3.6", - "dev": true, - "license": "MIT", - "dependencies": { - "get-func-name": "^2.0.0" - } - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/make-dir": { - "version": "3.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^6.0.0" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/make-error": { - "version": "1.3.6", - "dev": true, - "license": "ISC" - }, - "node_modules/media-typer": { - "version": "0.3.0", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/merge-descriptors": { - "version": "1.0.1", - "license": "MIT" - }, - "node_modules/merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true, - "engines": { - "node": ">= 8" - } - }, - "node_modules/methods": { - "version": "1.1.2", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/micromatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", - "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", - "dev": true, - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mime": { - "version": "1.6.0", - "license": "MIT", - "bin": { - "mime": "cli.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/mime-db": { - "version": "1.52.0", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mime-types": { - "version": "2.1.35", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", - "optional": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/minimatch": { - "version": "5.0.1", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/minimatch/node_modules/brace-expansion": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "devOptional": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "optional": true, - "dependencies": { - "minimist": "^1.2.6" - }, - "bin": { - "mkdirp": "bin/cmd.js" - } - }, - "node_modules/mocha": { - "version": "10.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.4", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "5.0.1", - "ms": "2.1.3", - "nanoid": "3.3.3", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "workerpool": "6.2.1", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" - }, - "bin": { - "_mocha": "bin/_mocha", - "mocha": "bin/mocha.js" - }, - "engines": { - "node": ">= 14.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/mochajs" - } - }, - "node_modules/mocha/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/mocha/node_modules/debug/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/mocha/node_modules/ms": { - "version": "2.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/moment": { - "version": "2.29.4", - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/moment-timezone": { - "version": "0.5.45", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz", - "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==", - "dependencies": { - "moment": "^2.29.4" - }, - "engines": { - "node": "*" - } - }, - "node_modules/ms": { - "version": "2.0.0", - "license": "MIT" - }, - "node_modules/multiparty": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/multiparty/-/multiparty-4.2.1.tgz", - "integrity": "sha512-AvESCnNoQlZiOfP9R4mxN8M9csy2L16EIbWIkt3l4FuGti9kXBS8QVzlfyg4HEnarJhrzZilgNFlZtqmoiAIIA==", - "dependencies": { - "fd-slicer": "1.1.0", - "http-errors": "~1.7.0", - "safe-buffer": "5.1.2", - "uid-safe": "2.1.5" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/multiparty/node_modules/depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/multiparty/node_modules/http-errors": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz", - "integrity": "sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==", - "dependencies": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/multiparty/node_modules/safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "node_modules/multiparty/node_modules/setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" - }, - "node_modules/multiparty/node_modules/statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/multiparty/node_modules/toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/nan": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", - "optional": true - }, - "node_modules/nanoid": { - "version": "3.3.3", - "dev": true, - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/napi-build-utils": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", - "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "optional": true - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true - }, - "node_modules/negotiator": { - "version": "0.6.3", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/nested-error-stacks": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.1.tgz", - "integrity": "sha512-9iN1ka/9zmX1ZvLV9ewJYEk9h7RyRRtqdK0woXcqohu8EWIerfPUjYJPg0ULy0UqP7cslmdGc8xKDJcojlKiaw==" - }, - "node_modules/nise": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.9.tgz", - "integrity": "sha512-qOnoujW4SV6e40dYxJOb3uvuoPHtmLzIk4TFo+j0jPJoC+5Z9xja5qH5JZobEPsa8+YYphMrOSwnrshEhG2qww==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^11.2.2", - "@sinonjs/text-encoding": "^0.7.2", - "just-extend": "^6.2.0", - "path-to-regexp": "^6.2.1" - } - }, - "node_modules/nise/node_modules/path-to-regexp": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", - "integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==", - "dev": true - }, - "node_modules/nock": { - "version": "13.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "debug": "^4.1.0", - "json-stringify-safe": "^5.0.1", - "lodash": "^4.17.21", - "propagate": "^2.0.0" - }, - "engines": { - "node": ">= 10.13" - } - }, - "node_modules/nock/node_modules/debug": { - "version": "4.3.4", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/nock/node_modules/ms": { - "version": "2.1.2", - "dev": true, - "license": "MIT" - }, - "node_modules/node-abi": { - "version": "2.30.1", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.1.tgz", - "integrity": "sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w==", - "optional": true, - "dependencies": { - "semver": "^5.4.1" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, - "node_modules/node-preload": { - "version": "0.2.1", - "dev": true, - "license": "MIT", - "dependencies": { - "process-on-spawn": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/node-releases": { - "version": "2.0.10", - "dev": true, - "license": "MIT" - }, - "node_modules/node-sql-parser": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/node-sql-parser/-/node-sql-parser-5.2.0.tgz", - "integrity": "sha512-lO/9ox0fLl47ksqlwGrorXm8BD+5UbxvUhj6T1XiCXnvoVwWWn7gC2l396p9OXNii19dc4MkyTN3XeJGNUZ8jw==", - "dependencies": { - "@types/pegjs": "^0.10.0", - "big-integer": "^1.6.48" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nodemon": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.0.1.tgz", - "integrity": "sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==", - "dev": true, - "dependencies": { - "chokidar": "^3.5.2", - "debug": "^3.2.7", - "ignore-by-default": "^1.0.1", - "minimatch": "^3.1.2", - "pstree.remy": "^1.1.8", - "semver": "^7.5.3", - "simple-update-notifier": "^2.0.0", - "supports-color": "^5.5.0", - "touch": "^3.1.0", - "undefsafe": "^2.0.5" - }, - "bin": { - "nodemon": "bin/nodemon.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/nodemon" - } - }, - "node_modules/nodemon/node_modules/debug": { - "version": "3.2.7", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" - } - }, - "node_modules/nodemon/node_modules/has-flag": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/nodemon/node_modules/minimatch": { - "version": "3.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/nodemon/node_modules/ms": { - "version": "2.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/nodemon/node_modules/supports-color": { - "version": "5.5.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/noop-logger": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", - "integrity": "sha512-6kM8CLXvuW5crTxsAtva2YLrRrDaiTIkIePWs9moLHqbFWT94WpNFjwS/5dfLfECg5i/lkmw3aoqVidxt23TEQ==", - "optional": true - }, - "node_modules/nopt": { - "version": "1.0.10", - "dev": true, - "license": "MIT", - "dependencies": { - "abbrev": "1" - }, - "bin": { - "nopt": "bin/nopt.js" - } - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "optional": true, - "dependencies": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "node_modules/number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/nyc": { - "version": "15.1.0", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "caching-transform": "^4.0.0", - "convert-source-map": "^1.7.0", - "decamelize": "^1.2.0", - "find-cache-dir": "^3.2.0", - "find-up": "^4.1.0", - "foreground-child": "^2.0.0", - "get-package-type": "^0.1.0", - "glob": "^7.1.6", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-hook": "^3.0.0", - "istanbul-lib-instrument": "^4.0.0", - "istanbul-lib-processinfo": "^2.0.2", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.0.2", - "make-dir": "^3.0.0", - "node-preload": "^0.2.1", - "p-map": "^3.0.0", - "process-on-spawn": "^1.0.0", - "resolve-from": "^5.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "spawn-wrap": "^2.0.0", - "test-exclude": "^6.0.0", - "yargs": "^15.0.2" - }, - "bin": { - "nyc": "bin/nyc.js" - }, - "engines": { - "node": ">=8.9" - } - }, - "node_modules/nyc/node_modules/camelcase": { - "version": "5.3.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/nyc/node_modules/cliui": { - "version": "6.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "node_modules/nyc/node_modules/decamelize": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/nyc/node_modules/find-up": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/locate-path": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/p-limit": { - "version": "2.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/nyc/node_modules/p-locate": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/wrap-ansi": { - "version": "6.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/y18n": { - "version": "4.0.3", - "dev": true, - "license": "ISC" - }, - "node_modules/nyc/node_modules/yargs": { - "version": "15.4.1", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/nyc/node_modules/yargs-parser": { - "version": "18.1.3", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/object-hash": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz", - "integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==", - "engines": { - "node": ">= 6" - } - }, - "node_modules/object-inspect": { - "version": "1.12.3", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/on-finished": { - "version": "2.4.1", - "license": "MIT", - "dependencies": { - "ee-first": "1.1.1" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "dependencies": { - "fn.name": "1.x.x" - } - }, - "node_modules/optional": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz", - "integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw==" - }, - "node_modules/optionator": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", - "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", - "dev": true, - "dependencies": { - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.5" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/os-homedir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", - "integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^3.0.2" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-map": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "aggregate-error": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/package-hash": { - "version": "4.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "graceful-fs": "^4.1.15", - "hasha": "^5.0.0", - "lodash.flattendeep": "^4.4.0", - "release-zalgo": "^1.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "dependencies": { - "callsites": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/parseurl": { - "version": "1.3.3", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "license": "MIT" - }, - "node_modules/path-to-regexp": { - "version": "0.1.7", - "license": "MIT" - }, - "node_modules/path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/pathval": { - "version": "1.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": "*" - } - }, - "node_modules/pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" - }, - "node_modules/pg": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.12.0.tgz", - "integrity": "sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==", - "dependencies": { - "pg-connection-string": "^2.6.4", - "pg-pool": "^3.6.2", - "pg-protocol": "^1.6.1", - "pg-types": "^2.1.0", - "pgpass": "1.x" - }, - "engines": { - "node": ">= 8.0.0" - }, - "optionalDependencies": { - "pg-cloudflare": "^1.1.1" - }, - "peerDependencies": { - "pg-native": ">=3.0.1" - }, - "peerDependenciesMeta": { - "pg-native": { - "optional": true - } - } - }, - "node_modules/pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", - "optional": true - }, - "node_modules/pg-connection-string": { - "version": "2.5.0", - "license": "MIT" - }, - "node_modules/pg-hstore": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz", - "integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==", - "dependencies": { - "underscore": "^1.13.1" - }, - "engines": { - "node": ">= 0.8.x" - } - }, - "node_modules/pg-int8": { - "version": "1.0.1", - "license": "ISC", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/pg-pool": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", - "peerDependencies": { - "pg": ">=8.0" - } - }, - "node_modules/pg-protocol": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", - "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" - }, - "node_modules/pg-types": { - "version": "2.2.0", - "license": "MIT", - "dependencies": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/pg/node_modules/pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, - "node_modules/pgpass": { - "version": "1.0.5", - "license": "MIT", - "dependencies": { - "split2": "^4.1.0" - } - }, - "node_modules/picocolors": { - "version": "1.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "2.3.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/find-up": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/locate-path": { - "version": "5.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pkg-dir/node_modules/p-limit": { - "version": "2.3.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/pkg-dir/node_modules/p-locate": { - "version": "4.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/postgres-array": { - "version": "2.0.0", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/postgres-bytea": { - "version": "1.0.0", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-date": { - "version": "1.0.7", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/postgres-interval": { - "version": "1.2.0", - "license": "MIT", - "dependencies": { - "xtend": "^4.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/prebuild-install": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz", - "integrity": "sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg==", - "optional": true, - "dependencies": { - "detect-libc": "^1.0.3", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "napi-build-utils": "^1.0.1", - "node-abi": "^2.7.0", - "noop-logger": "^0.1.1", - "npmlog": "^4.0.1", - "os-homedir": "^1.0.1", - "pump": "^2.0.1", - "rc": "^1.2.7", - "simple-get": "^2.7.0", - "tar-fs": "^1.13.0", - "tunnel-agent": "^0.6.0", - "which-pm-runs": "^1.0.0" - }, - "bin": { - "prebuild-install": "bin.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/process-nextick-args": { - "version": "2.0.1", - "license": "MIT" - }, - "node_modules/process-on-spawn": { - "version": "1.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "fromentries": "^1.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/prom-client": { - "version": "14.2.0", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.2.0.tgz", - "integrity": "sha512-sF308EhTenb/pDRPakm+WgiN+VdM/T1RaHj1x+MvAuT8UiQP8JmOEbxVqtkbfR4LrvOg5n7ic01kRBDGXjYikA==", - "dependencies": { - "tdigest": "^0.1.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/propagate": { - "version": "2.0.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 8" - } - }, - "node_modules/proxy-addr": { - "version": "2.0.7", - "license": "MIT", - "dependencies": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - }, - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/proxy-from-env": { - "version": "1.1.0", - "license": "MIT" - }, - "node_modules/pstree.remy": { - "version": "1.1.8", - "dev": true, - "license": "MIT" - }, - "node_modules/pump": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", - "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", - "optional": true, - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/punycode": { - "version": "2.3.0", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/qs": { - "version": "6.11.0", - "license": "BSD-3-Clause", - "dependencies": { - "side-channel": "^1.0.4" - }, - "engines": { - "node": ">=0.6" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", - "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", - "engines": { - "node": ">=0.4.x" - } - }, - "node_modules/queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] - }, - "node_modules/random-bytes": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", - "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/randombytes": { - "version": "2.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "safe-buffer": "^5.1.0" - } - }, - "node_modules/range-parser": { - "version": "1.2.1", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", - "dependencies": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "optional": true, - "dependencies": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "bin": { - "rc": "cli.js" - } - }, - "node_modules/rc/node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "optional": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/readable-stream": { - "version": "2.3.8", - "license": "MIT", - "dependencies": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - } - }, - "node_modules/readable-stream/node_modules/safe-buffer": { - "version": "5.1.2", - "license": "MIT" - }, - "node_modules/readdirp": { - "version": "3.6.0", - "dev": true, - "license": "MIT", - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "node_modules/rechoir": { - "version": "0.8.0", - "license": "MIT", - "dependencies": { - "resolve": "^1.20.0" - }, - "engines": { - "node": ">= 10.13.0" - } - }, - "node_modules/release-zalgo": { - "version": "1.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "es6-error": "^4.0.1" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/require-directory": { - "version": "2.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-from-string": { - "version": "2.0.2", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/require-main-filename": { - "version": "2.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/resolve": { - "version": "1.22.1", - "license": "MIT", - "dependencies": { - "is-core-module": "^2.9.0", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", - "engines": { - "node": ">= 4" - } - }, - "node_modules/retry-as-promised": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-7.0.4.tgz", - "integrity": "sha512-XgmCoxKWkDofwH8WddD0w85ZfqYz+ZHlr5yo+3YUCfycWawU56T5ckWXsScsj5B8tqUcIG67DxXByo3VUgiAdA==" - }, - "node_modules/retry-request": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", - "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", - "dependencies": { - "@types/request": "^2.48.8", - "extend": "^3.0.2", - "teeny-request": "^9.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true, - "engines": { - "iojs": ">=1.0.0", - "node": ">=0.10.0" - } - }, - "node_modules/rfdc": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.1.tgz", - "integrity": "sha512-r5a3l5HzYlIC68TpmYKlxWjmOP6wiPJ1vWv2HeLhNsRZMrCkxeqxiHlQ21oXmQ4F3SiryXBHhAD7JZqvOJjFmg==" - }, - "node_modules/rimraf": { - "version": "3.0.2", - "dev": true, - "license": "ISC", - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "queue-microtask": "^1.2.2" - } - }, - "node_modules/safe-buffer": { - "version": "5.2.1", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT" - }, - "node_modules/safe-stable-stringify": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", - "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==", - "engines": { - "node": ">=10" - } - }, - "node_modules/safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "node_modules/sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" - }, - "node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, - "node_modules/send": { - "version": "0.18.0", - "license": "MIT", - "dependencies": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/send/node_modules/ms": { - "version": "2.1.3", - "license": "MIT" - }, - "node_modules/sequelize": { - "version": "6.37.3", - "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.3.tgz", - "integrity": "sha512-V2FTqYpdZjPy3VQrZvjTPnOoLm0KudCRXfGWp48QwhyPPp2yW8z0p0sCYZd/em847Tl2dVxJJ1DR+hF+O77T7A==", - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/sequelize" - } - ], - "dependencies": { - "@types/debug": "^4.1.8", - "@types/validator": "^13.7.17", - "debug": "^4.3.4", - "dottie": "^2.0.6", - "inflection": "^1.13.4", - "lodash": "^4.17.21", - "moment": "^2.29.4", - "moment-timezone": "^0.5.43", - "pg-connection-string": "^2.6.1", - "retry-as-promised": "^7.0.4", - "semver": "^7.5.4", - "sequelize-pool": "^7.1.0", - "toposort-class": "^1.0.1", - "uuid": "^8.3.2", - "validator": "^13.9.0", - "wkx": "^0.5.0" - }, - "engines": { - "node": ">=10.0.0" - }, - "peerDependenciesMeta": { - "ibm_db": { - "optional": true - }, - "mariadb": { - "optional": true - }, - "mysql2": { - "optional": true - }, - "oracledb": { - "optional": true - }, - "pg": { - "optional": true - }, - "pg-hstore": { - "optional": true - }, - "snowflake-sdk": { - "optional": true - }, - "sqlite3": { - "optional": true - }, - "tedious": { - "optional": true - } - } - }, - "node_modules/sequelize-pool": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-7.1.0.tgz", - "integrity": "sha512-G9c0qlIWQSK29pR/5U2JF5dDQeqqHRragoyahj/Nx4KOOQ3CPPfzxnfqFPCSB7x5UgjOgnZ61nSxz+fjDpRlJg==", - "engines": { - "node": ">= 10.0.0" - } - }, - "node_modules/sequelize/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/sequelize/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/sequelize/node_modules/pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, - "node_modules/sequelize/node_modules/uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/serialize-javascript": { - "version": "6.0.0", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "randombytes": "^2.1.0" - } - }, - "node_modules/serve-static": { - "version": "1.15.0", - "license": "MIT", - "dependencies": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.18.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/set-blocking": { - "version": "2.0.0", - "devOptional": true, - "license": "ISC" - }, - "node_modules/setprototypeof": { - "version": "1.2.0", - "license": "ISC" - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/side-channel": { - "version": "1.0.4", - "license": "MIT", - "dependencies": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "devOptional": true, - "license": "ISC" - }, - "node_modules/simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "optional": true - }, - "node_modules/simple-get": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.2.tgz", - "integrity": "sha512-Ijd/rV5o+mSBBs4F/x9oDPtTx9Zb6X9brmnXvMW4J7IR15ngi9q5xxqWBKU744jTZiaXtxaPL7uHG6vtN8kUkw==", - "optional": true, - "dependencies": { - "decompress-response": "^3.3.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "node_modules/simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "dependencies": { - "is-arrayish": "^0.3.1" - } - }, - "node_modules/simple-update-notifier": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", - "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", - "dev": true, - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sinon": { - "version": "17.0.1", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-17.0.1.tgz", - "integrity": "sha512-wmwE19Lie0MLT+ZYNpDymasPHUKTaZHUH/pKEubRXIzySv9Atnlw+BUMGCzWgV7b7wO+Hw6f1TEOr0IUnmU8/g==", - "dev": true, - "dependencies": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^11.2.2", - "@sinonjs/samsam": "^8.0.0", - "diff": "^5.1.0", - "nise": "^5.1.5", - "supports-color": "^7.2.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/sinon" - } - }, - "node_modules/sinon/node_modules/diff": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", - "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", - "dev": true, - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/sinon/node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/slug": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/slug/-/slug-9.1.0.tgz", - "integrity": "sha512-ioOsCfzQSu+D6NZ8XMCR8IW9FgvF8W7Xzz56hBkB/ALvNaWeBs2MUvvPugq3GCrxfHPFeK6hAxGkY/WLnfX2Lg==", - "bin": { - "slug": "cli.js" - } - }, - "node_modules/snappy": { - "version": "6.3.5", - "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.3.5.tgz", - "integrity": "sha512-lonrUtdp1b1uDn1dbwgQbBsb5BbaiLeKq+AGwOk2No+en+VvJThwmtztwulEQsLinRF681pBqib0NUZaizKLIA==", - "hasInstallScript": true, - "optional": true, - "dependencies": { - "bindings": "^1.3.1", - "nan": "^2.14.1", - "prebuild-install": "5.3.0" - } - }, - "node_modules/snappyjs": { - "version": "0.6.1", - "license": "MIT" - }, - "node_modules/source-map": { - "version": "0.6.1", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/spawn-wrap": { - "version": "2.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "foreground-child": "^2.0.0", - "is-windows": "^1.0.2", - "make-dir": "^3.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "which": "^2.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/split2": { - "version": "4.1.0", - "license": "ISC", - "engines": { - "node": ">= 10.x" - } - }, - "node_modules/stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", - "engines": { - "node": "*" - } - }, - "node_modules/statuses": { - "version": "2.0.1", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/stream-browserify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", - "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", - "dependencies": { - "inherits": "~2.0.4", - "readable-stream": "^3.5.0" - } - }, - "node_modules/stream-browserify/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/stream-events": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", - "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", - "dependencies": { - "stubs": "^3.0.0" - } - }, - "node_modules/stream-shift": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" - }, - "node_modules/streamroller": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-3.1.5.tgz", - "integrity": "sha512-KFxaM7XT+irxvdqSP1LGLgNWbYN7ay5owZ3r/8t77p+EtSUAfUgtl7be3xtqtOmGUl9K9YPO2ca8133RlTjvKw==", - "dependencies": { - "date-format": "^4.0.14", - "debug": "^4.3.4", - "fs-extra": "^8.1.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/streamroller/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/streamroller/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/string_decoder": { - "version": "1.1.1", - "license": "MIT", - "dependencies": { - "safe-buffer": "~5.1.0" - } - }, - "node_modules/string_decoder/node_modules/safe-buffer": { - "version": "5.1.2", - "license": "MIT" - }, - "node_modules/string-width": { - "version": "4.2.3", - "devOptional": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "devOptional": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-bom": { - "version": "4.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" - }, - "node_modules/stubs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", - "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" - }, - "node_modules/superagent": { - "version": "3.8.3", - "dev": true, - "license": "MIT", - "dependencies": { - "component-emitter": "^1.2.0", - "cookiejar": "^2.1.0", - "debug": "^3.1.0", - "extend": "^3.0.0", - "form-data": "^2.3.1", - "formidable": "^1.2.0", - "methods": "^1.1.1", - "mime": "^1.4.1", - "qs": "^6.5.1", - "readable-stream": "^2.3.5" - }, - "engines": { - "node": ">= 4.0" - } - }, - "node_modules/superagent/node_modules/debug": { - "version": "3.2.7", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.1" - } - }, - "node_modules/superagent/node_modules/form-data": { - "version": "2.5.1", - "dev": true, - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 0.12" - } - }, - "node_modules/superagent/node_modules/ms": { - "version": "2.1.3", - "dev": true, - "license": "MIT" - }, - "node_modules/supports-color": { - "version": "8.1.1", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/tar-fs": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", - "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", - "optional": true, - "dependencies": { - "chownr": "^1.0.1", - "mkdirp": "^0.5.1", - "pump": "^1.0.0", - "tar-stream": "^1.1.2" - } - }, - "node_modules/tar-fs/node_modules/pump": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", - "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", - "optional": true, - "dependencies": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "node_modules/tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", - "optional": true, - "dependencies": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", - "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/tar-stream/node_modules/bl": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", - "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", - "optional": true, - "dependencies": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, - "node_modules/tarn": { - "version": "3.0.2", - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/tdigest": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", - "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", - "dependencies": { - "bintrees": "1.0.2" - } - }, - "node_modules/teeny-request": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", - "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", - "dependencies": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.9", - "stream-events": "^1.0.5", - "uuid": "^9.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/teeny-request/node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, - "node_modules/teeny-request/node_modules/debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/teeny-request/node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/teeny-request/node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/teeny-request/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "node_modules/teeny-request/node_modules/uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "bin": { - "uuid": "dist/bin/uuid" - } - }, - "node_modules/test-exclude": { - "version": "6.0.0", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/test-exclude/node_modules/minimatch": { - "version": "3.1.2", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, - "node_modules/text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true - }, - "node_modules/tildify": { - "version": "2.0.0", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", - "optional": true - }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/toidentifier": { - "version": "1.0.1", - "license": "MIT", - "engines": { - "node": ">=0.6" - } - }, - "node_modules/toposort-class": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", - "integrity": "sha512-OsLcGGbYF3rMjPUf8oKktyvCiUxSbqMMS39m33MAjLTC1DVIH6x3WSt63/M77ihI09+Sdfk1AXvfhCEeUmC7mg==" - }, - "node_modules/touch": { - "version": "3.1.0", - "dev": true, - "license": "ISC", - "dependencies": { - "nopt": "~1.0.10" - }, - "bin": { - "nodetouch": "bin/nodetouch.js" - } - }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "node_modules/traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==", - "engines": { - "node": "*" - } - }, - "node_modules/trino-client": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/trino-client/-/trino-client-0.2.2.tgz", - "integrity": "sha512-TMndAbFiiGlAbJotsqsbMSqFZ8sPtE+KBIl/qYEAI9+eAXruNvz0BhxFC67PE39eouVn+JtF6L8Yr2q6/sAlAA==", - "dependencies": { - "axios": "1.6.2" - } - }, - "node_modules/triple-beam": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", - "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", - "engines": { - "node": ">= 14.0.0" - } - }, - "node_modules/ts-api-utils": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", - "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", - "dev": true, - "engines": { - "node": ">=16" - }, - "peerDependencies": { - "typescript": ">=4.2.0" - } - }, - "node_modules/ts-node": { - "version": "10.9.1", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-cwd": "dist/bin-cwd.js", - "ts-node-esm": "dist/bin-esm.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "peerDependencies": { - "@swc/core": ">=1.2.50", - "@swc/wasm": ">=1.2.50", - "@types/node": "*", - "typescript": ">=2.7" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "@swc/wasm": { - "optional": true - } - } - }, - "node_modules/ts-node/node_modules/diff": { - "version": "4.0.2", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/tsconfig-paths": { - "version": "4.1.2", - "dev": true, - "license": "MIT", - "dependencies": { - "json5": "^2.2.2", - "minimist": "^1.2.6", - "strip-bom": "^3.0.0" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/tsconfig-paths/node_modules/strip-bom": { - "version": "3.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" - }, - "node_modules/tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "optional": true, - "dependencies": { - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": "*" - } - }, - "node_modules/type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "dependencies": { - "prelude-ls": "^1.2.1" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/type-detect": { - "version": "4.0.8", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/type-fest": { - "version": "0.8.1", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=8" - } - }, - "node_modules/type-is": { - "version": "1.6.18", - "license": "MIT", - "dependencies": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/typedarray-to-buffer": { - "version": "3.1.5", - "dev": true, - "license": "MIT", - "dependencies": { - "is-typedarray": "^1.0.0" - } - }, - "node_modules/typescript": { - "version": "4.9.5", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=4.2.0" - } - }, - "node_modules/uid-safe": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", - "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==", - "dependencies": { - "random-bytes": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/undefsafe": { - "version": "2.0.5", - "dev": true, - "license": "MIT" - }, - "node_modules/underscore": { - "version": "1.13.6", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", - "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" - }, - "node_modules/universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", - "engines": { - "node": ">= 4.0.0" - } - }, - "node_modules/unpipe": { - "version": "1.0.0", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/update-browserslist-db": { - "version": "1.0.10", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.1.1", - "picocolors": "^1.0.0" - }, - "bin": { - "browserslist-lint": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/uri-js": { - "version": "4.4.1", - "license": "BSD-2-Clause", - "dependencies": { - "punycode": "^2.1.0" - } - }, - "node_modules/url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", - "dependencies": { - "punycode": "1.3.2", - "querystring": "0.2.0" - } - }, - "node_modules/url/node_modules/punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" - }, - "node_modules/util": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", - "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", - "dependencies": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, - "node_modules/util-deprecate": { - "version": "1.0.2", - "license": "MIT" - }, - "node_modules/utils-merge": { - "version": "1.0.1", - "license": "MIT", - "engines": { - "node": ">= 0.4.0" - } - }, - "node_modules/uuid": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, - "node_modules/v8-compile-cache-lib": { - "version": "3.0.1", - "dev": true, - "license": "MIT" - }, - "node_modules/validator": { - "version": "13.12.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.12.0.tgz", - "integrity": "sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==", - "engines": { - "node": ">= 0.10" - } - }, - "node_modules/vary": { - "version": "1.1.2", - "license": "MIT", - "engines": { - "node": ">= 0.8" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "node_modules/which": { - "version": "2.0.2", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/which-module": { - "version": "2.0.0", - "dev": true, - "license": "ISC" - }, - "node_modules/which-pm-runs": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.1.0.tgz", - "integrity": "sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==", - "optional": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/which-typed-array": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", - "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==", - "dependencies": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/wide-align": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", - "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", - "optional": true, - "dependencies": { - "string-width": "^1.0.2 || 2 || 3 || 4" - } - }, - "node_modules/winston": { - "version": "2.4.7", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.7.tgz", - "integrity": "sha512-vLB4BqzCKDnnZH9PHGoS2ycawueX4HLqENXQitvFHczhgW2vFpSOn31LZtVr1KU8YTw7DS4tM+cqyovxo8taVg==", - "dependencies": { - "async": "^2.6.4", - "colors": "1.0.x", - "cycle": "1.0.x", - "eyes": "0.1.x", - "isstream": "0.1.x", - "stack-trace": "0.0.x" - }, - "engines": { - "node": ">= 0.10.0" - } - }, - "node_modules/winston-compat": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/winston-compat/-/winston-compat-0.1.5.tgz", - "integrity": "sha512-EPvPcHT604AV3Ji6d3+vX8ENKIml9VSxMRnPQ+cuK/FX6f3hvPP2hxyoeeCOCFvDrJEujalfcKWlWPvAnFyS9g==", - "dependencies": { - "cycle": "~1.0.3", - "logform": "^1.6.0", - "triple-beam": "^1.2.0" - }, - "engines": { - "node": ">= 6.4.0" - } - }, - "node_modules/winston-daily-rotate-file": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-3.2.3.tgz", - "integrity": "sha512-BOvmvQH2WaiexOjzj14YNHSc18IDyZJ9t4pMsbTERjpjMltoBVijM8DDJJPr2jSqELSNnbgGPBk3kDQSRgOAtQ==", - "dependencies": { - "file-stream-rotator": "^0.2.1", - "semver": "^5.5.0", - "triple-beam": "^1.3.0", - "winston-compat": "^0.1.4", - "winston-transport": "^4.2.0" - }, - "engines": { - "node": ">=6" - }, - "peerDependencies": { - "winston": "^2 || ^3" - } - }, - "node_modules/winston-transport": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.7.0.tgz", - "integrity": "sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg==", - "dependencies": { - "logform": "^2.3.2", - "readable-stream": "^3.6.0", - "triple-beam": "^1.3.0" - }, - "engines": { - "node": ">= 12.0.0" - } - }, - "node_modules/winston-transport/node_modules/@colors/colors": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", - "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==", - "engines": { - "node": ">=0.1.90" - } - }, - "node_modules/winston-transport/node_modules/fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" - }, - "node_modules/winston-transport/node_modules/logform": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.5.1.tgz", - "integrity": "sha512-9FyqAm9o9NKKfiAKfZoYo9bGXXuwMkxQiQttkT4YjjVtQVIQtK6LmVtlxmCaFswo6N4AfEkHqZTV0taDtPotNg==", - "dependencies": { - "@colors/colors": "1.5.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - } - }, - "node_modules/winston-transport/node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "node_modules/winston-transport/node_modules/readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "dependencies": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/wkx": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.5.0.tgz", - "integrity": "sha512-Xng/d4Ichh8uN4l0FToV/258EjMGU9MGcA0HV2d9B/ZpZB3lqQm7nkOdZdm5GhKtLLhAE7PiVQwN4eN+2YJJUg==", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/word-wrap": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", - "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/workerpool": { - "version": "6.2.1", - "dev": true, - "license": "Apache-2.0" - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "license": "ISC" - }, - "node_modules/write-file-atomic": { - "version": "3.0.3", - "dev": true, - "license": "ISC", - "dependencies": { - "imurmurhash": "^0.1.4", - "is-typedarray": "^1.0.0", - "signal-exit": "^3.0.2", - "typedarray-to-buffer": "^3.1.5" - } - }, - "node_modules/xml2js": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", - "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", - "dependencies": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - }, - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", - "engines": { - "node": ">=4.0" - } - }, - "node_modules/xtend": { - "version": "4.0.2", - "license": "MIT", - "engines": { - "node": ">=0.4" - } - }, - "node_modules/y18n": { - "version": "5.0.8", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "dev": true, - "license": "ISC" - }, - "node_modules/yargs": { - "version": "16.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-parser": { - "version": "20.2.4", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yargs-unparser": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "dependencies": { - "camelcase": "^6.0.0", - "decamelize": "^4.0.0", - "flat": "^5.0.2", - "is-plain-obj": "^2.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/yn": { - "version": "3.1.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - } - }, - "dependencies": { - "@ampproject/remapping": { - "version": "2.2.0", - "dev": true, - "requires": { - "@jridgewell/gen-mapping": "^0.1.0", - "@jridgewell/trace-mapping": "^0.3.9" - } - }, - "@aws-crypto/crc32": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-3.0.0.tgz", - "integrity": "sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA==", - "requires": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/crc32c": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/crc32c/-/crc32c-3.0.0.tgz", - "integrity": "sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w==", - "requires": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/ie11-detection": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz", - "integrity": "sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q==", - "requires": { - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/sha1-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha1-browser/-/sha1-browser-3.0.0.tgz", - "integrity": "sha512-NJth5c997GLHs6nOYTzFKTbYdMNA6/1XlKVgnZoaZcQ7z7UJlOgj2JdbHE8tiYLS3fzXNCguct77SPGat2raSw==", - "requires": { - "@aws-crypto/ie11-detection": "^3.0.0", - "@aws-crypto/supports-web-crypto": "^3.0.0", - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/sha256-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz", - "integrity": "sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ==", - "requires": { - "@aws-crypto/ie11-detection": "^3.0.0", - "@aws-crypto/sha256-js": "^3.0.0", - "@aws-crypto/supports-web-crypto": "^3.0.0", - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-locate-window": "^3.0.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/sha256-js": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz", - "integrity": "sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ==", - "requires": { - "@aws-crypto/util": "^3.0.0", - "@aws-sdk/types": "^3.222.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/supports-web-crypto": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz", - "integrity": "sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg==", - "requires": { - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-crypto/util": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-3.0.0.tgz", - "integrity": "sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w==", - "requires": { - "@aws-sdk/types": "^3.222.0", - "@aws-sdk/util-utf8-browser": "^3.0.0", - "tslib": "^1.11.1" - }, - "dependencies": { - "tslib": { - "version": "1.14.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", - "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==" - } - } - }, - "@aws-sdk/client-cognito-identity": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-cognito-identity/-/client-cognito-identity-3.427.0.tgz", - "integrity": "sha512-9brRaNnl6haE7R3R43A5CSNw0k1YtB3xjuArbMg/p6NDUpvRSRgOVNWu2R02Yjh/j2ZuaLOCPLuCipb+PHQPKQ==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.427.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/client-s3": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-s3/-/client-s3-3.590.0.tgz", - "integrity": "sha512-so+pNua0ihsHaSdskw8HCwruoYTAfYSEs3ix4GD1++83C96KaJp3udAutYiCA+84JXg9zitFa7eK7ORJAVZmTw==", - "requires": { - "@aws-crypto/sha1-browser": "3.0.0", - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/client-sts": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-bucket-endpoint": "3.587.0", - "@aws-sdk/middleware-expect-continue": "3.577.0", - "@aws-sdk/middleware-flexible-checksums": "3.587.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-location-constraint": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-sdk-s3": "3.587.0", - "@aws-sdk/middleware-signing": "3.587.0", - "@aws-sdk/middleware-ssec": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/signature-v4-multi-region": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@aws-sdk/xml-builder": "3.575.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/eventstream-serde-browser": "^3.0.0", - "@smithy/eventstream-serde-config-resolver": "^3.0.0", - "@smithy/eventstream-serde-node": "^3.0.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-blob-browser": "^3.0.0", - "@smithy/hash-node": "^3.0.0", - "@smithy/hash-stream-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/md5-js": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "@smithy/util-utf8": "^3.0.0", - "@smithy/util-waiter": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/client-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.590.0.tgz", - "integrity": "sha512-6xbC6oQVJKBRTyXyR3C15ksUsPOyW4p+uCj7dlKYWGJvh4vGTV8KhZKS53oPG8t4f1+OMJWjr5wKuXRoaFsmhQ==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/client-sts": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.590.0.tgz", - "integrity": "sha512-f4R1v1LSn4uLYZ5qj4DyL6gp7PXXzJeJsm2seheiJX+53LSF5L7XSDnQVtX1p9Tevv0hp2YUWUTg6QYwIVSuGg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-env": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz", - "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-http": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.587.0.tgz", - "integrity": "sha512-Su1SRWVRCuR1e32oxX3C1V4c5hpPN20WYcRfdcr2wXwHqSvys5DrnmuCC+JoEnS/zt3adUJhPliTqpfKgSdMrA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-ini": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.590.0.tgz", - "integrity": "sha512-Y5cFciAK38VIvRgZeND7HvFNR32thGtQb8Xop6cMn33FC78uwcRIu9Hc9699XTclCZqz4+Xl1WU+dZ+rnFn2AA==", - "requires": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-node": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.590.0.tgz", - "integrity": "sha512-Ky38mNFoXobGrDQ11P3dU1e+q1nRJ7eZl8l15KUpvZCe/hOudbxQi/epQrCazD/gRYV2fTyczdLlZzB5ZZ8DhQ==", - "requires": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-ini": "3.590.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-process": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz", - "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.590.0.tgz", - "integrity": "sha512-v+0j/I+je9okfwXsgmLppmwIE+TuMp5WqLz7r7PHz9KjzLyKaKTDvfllFD+8oPpBqnmOWiJ9qTGPkrfhB7a/fQ==", - "requires": { - "@aws-sdk/client-sso": "3.590.0", - "@aws-sdk/token-providers": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-web-identity": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz", - "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-host-header": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz", - "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-logger": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz", - "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-recursion-detection": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz", - "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-signing": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.587.0.tgz", - "integrity": "sha512-tiZaTDj4RvhXGRAlncFn7CSEfL3iNPO67WSaxAq+Ls5j1VgczPhu5262cWONNoMgth3nXR1hhLC4ITSl/a6AzA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-user-agent": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz", - "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/region-config-resolver": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz", - "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/token-providers": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz", - "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-endpoints": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz", - "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "@smithy/util-endpoints": "^2.0.1", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-user-agent-browser": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz", - "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-user-agent-node": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz", - "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/config-resolver": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.1.tgz", - "integrity": "sha512-hbkYJc20SBDz2qqLzttjI/EqXemtmWk0ooRznLsiXp3066KQRTvuKHa7U4jCZCJq6Dozqvy0R1/vNESC9inPJg==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/credential-provider-imds": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz", - "integrity": "sha512-q4A4d38v8pYYmseu/jTS3Z5I3zXlEOe5Obi+EJreVKgSVyWUHOd7/yaVCinC60QG4MRyCs98tcxBH1IMC0bu7Q==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/hash-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.0.tgz", - "integrity": "sha512-84qXstNemP3XS5jcof0el6+bDfjzuvhJPQTEfro3lgtbCtKgzPm3MgiS6ehXVPjeQ5+JS0HqmTz8f/RYfzHVxw==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/invalid-dependency": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.0.tgz", - "integrity": "sha512-F6wBBaEFgJzj0s4KUlliIGPmqXemwP6EavgvDqYwCH40O5Xr2iMHvS8todmGVZtuJCorBkXsYLyTu4PuizVq5g==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-content-length": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.0.tgz", - "integrity": "sha512-3C4s4d/iGobgCtk2tnWW6+zSTOBg1PRAm2vtWZLdriwTroFbbWNSr3lcyzHdrQHnEXYCC5K52EbpfodaIUY8sg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "requires": { - "@smithy/types": "^3.0.0" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-body-length-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", - "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-body-length-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", - "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-defaults-mode-browser": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.3.tgz", - "integrity": "sha512-3DFON2bvXJAukJe+qFgPV/rorG7ZD3m4gjCXHD1V5z/tgKQp5MCTCLntrd686tX6tj8Uli3lefWXJudNg5WmCA==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-defaults-mode-node": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.3.tgz", - "integrity": "sha512-D0b8GJXecT00baoSQ3Iieu3k3mZ7GY8w1zmg8pdogYrGvWJeLcIclqk2gbkG4K0DaBGWrO6v6r20iwIFfDYrmA==", - "requires": { - "@smithy/config-resolver": "^3.0.1", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "requires": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "@aws-sdk/client-sso": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.427.0.tgz", - "integrity": "sha512-sFVFEmsQ1rmgYO1SgrOTxE/MTKpeE4hpOkm1WqhLQK7Ij136vXpjCxjH1JYZiHiUzO1wr9t4ex4dlB5J3VS/Xg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/client-sso-oidc": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso-oidc/-/client-sso-oidc-3.590.0.tgz", - "integrity": "sha512-3yCLPjq6WFfDpdUJKk/gSz4eAPDTjVknXaveMPi2QoVBCshneOnJsV16uNKlpVF1frTHrrDRfKYmbaVh6nFBvQ==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sts": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/client-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sso/-/client-sso-3.590.0.tgz", - "integrity": "sha512-6xbC6oQVJKBRTyXyR3C15ksUsPOyW4p+uCj7dlKYWGJvh4vGTV8KhZKS53oPG8t4f1+OMJWjr5wKuXRoaFsmhQ==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/client-sts": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.590.0.tgz", - "integrity": "sha512-f4R1v1LSn4uLYZ5qj4DyL6gp7PXXzJeJsm2seheiJX+53LSF5L7XSDnQVtX1p9Tevv0hp2YUWUTg6QYwIVSuGg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/client-sso-oidc": "3.590.0", - "@aws-sdk/core": "3.588.0", - "@aws-sdk/credential-provider-node": "3.590.0", - "@aws-sdk/middleware-host-header": "3.577.0", - "@aws-sdk/middleware-logger": "3.577.0", - "@aws-sdk/middleware-recursion-detection": "3.577.0", - "@aws-sdk/middleware-user-agent": "3.587.0", - "@aws-sdk/region-config-resolver": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@aws-sdk/util-user-agent-browser": "3.577.0", - "@aws-sdk/util-user-agent-node": "3.587.0", - "@smithy/config-resolver": "^3.0.1", - "@smithy/core": "^2.1.1", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/hash-node": "^3.0.0", - "@smithy/invalid-dependency": "^3.0.0", - "@smithy/middleware-content-length": "^3.0.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-body-length-browser": "^3.0.0", - "@smithy/util-body-length-node": "^3.0.0", - "@smithy/util-defaults-mode-browser": "^3.0.3", - "@smithy/util-defaults-mode-node": "^3.0.3", - "@smithy/util-endpoints": "^2.0.1", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-env": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.587.0.tgz", - "integrity": "sha512-Hyg/5KFECIk2k5o8wnVEiniV86yVkhn5kzITUydmNGCkXdBFHMHRx6hleQ1bqwJHbBskyu8nbYamzcwymmGwmw==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-http": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.587.0.tgz", - "integrity": "sha512-Su1SRWVRCuR1e32oxX3C1V4c5hpPN20WYcRfdcr2wXwHqSvys5DrnmuCC+JoEnS/zt3adUJhPliTqpfKgSdMrA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-ini": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.590.0.tgz", - "integrity": "sha512-Y5cFciAK38VIvRgZeND7HvFNR32thGtQb8Xop6cMn33FC78uwcRIu9Hc9699XTclCZqz4+Xl1WU+dZ+rnFn2AA==", - "requires": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-node": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.590.0.tgz", - "integrity": "sha512-Ky38mNFoXobGrDQ11P3dU1e+q1nRJ7eZl8l15KUpvZCe/hOudbxQi/epQrCazD/gRYV2fTyczdLlZzB5ZZ8DhQ==", - "requires": { - "@aws-sdk/credential-provider-env": "3.587.0", - "@aws-sdk/credential-provider-http": "3.587.0", - "@aws-sdk/credential-provider-ini": "3.590.0", - "@aws-sdk/credential-provider-process": "3.587.0", - "@aws-sdk/credential-provider-sso": "3.590.0", - "@aws-sdk/credential-provider-web-identity": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-process": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.587.0.tgz", - "integrity": "sha512-V4xT3iCqkF8uL6QC4gqBJg/2asd/damswP1h9HCfqTllmPWzImS+8WD3VjgTLw5b0KbTy+ZdUhKc0wDnyzkzxg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-sso": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.590.0.tgz", - "integrity": "sha512-v+0j/I+je9okfwXsgmLppmwIE+TuMp5WqLz7r7PHz9KjzLyKaKTDvfllFD+8oPpBqnmOWiJ9qTGPkrfhB7a/fQ==", - "requires": { - "@aws-sdk/client-sso": "3.590.0", - "@aws-sdk/token-providers": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/credential-provider-web-identity": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.587.0.tgz", - "integrity": "sha512-XqIx/I2PG7kyuw3WjAP9wKlxy8IvFJwB8asOFT1xPFoVfZYKIogjG9oLP5YiRtfvDkWIztHmg5MlVv3HdJDGRw==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-host-header": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.577.0.tgz", - "integrity": "sha512-9ca5MJz455CODIVXs0/sWmJm7t3QO4EUa1zf8pE8grLpzf0J94bz/skDWm37Pli13T3WaAQBHCTiH2gUVfCsWg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-logger": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.577.0.tgz", - "integrity": "sha512-aPFGpGjTZcJYk+24bg7jT4XdIp42mFXSuPt49lw5KygefLyJM/sB0bKKqPYYivW0rcuZ9brQ58eZUNthrzYAvg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-recursion-detection": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.577.0.tgz", - "integrity": "sha512-pn3ZVEd2iobKJlR3H+bDilHjgRnNrQ6HMmK9ZzZw89Ckn3Dcbv48xOv4RJvu0aU8SDLl/SNCxppKjeLDTPGBNA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/middleware-user-agent": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.587.0.tgz", - "integrity": "sha512-SyDomN+IOrygLucziG7/nOHkjUXES5oH5T7p8AboO8oakMQJdnudNXiYWTicQWO52R51U6CR27rcMPTGeMedYA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-endpoints": "3.587.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/region-config-resolver": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.587.0.tgz", - "integrity": "sha512-93I7IPZtulZQoRK+O20IJ4a1syWwYPzoO2gc3v+/GNZflZPV3QJXuVbIm0pxBsu0n/mzKGUKqSOLPIaN098HcQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/token-providers": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.587.0.tgz", - "integrity": "sha512-ULqhbnLy1hmJNRcukANBWJmum3BbjXnurLPSFXoGdV0llXYlG55SzIla2VYqdveQEEjmsBuTZdFvXAtNpmS5Zg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-endpoints": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.587.0.tgz", - "integrity": "sha512-8I1HG6Em8wQWqKcRW6m358mqebRVNpL8XrrEoT4In7xqkKkmYtHRNVYP6lcmiQh5pZ/c/FXu8dSchuFIWyEtqQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "@smithy/util-endpoints": "^2.0.1", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-user-agent-browser": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.577.0.tgz", - "integrity": "sha512-zEAzHgR6HWpZOH7xFgeJLc6/CzMcx4nxeQolZxVZoB5pPaJd3CjyRhZN0xXeZB0XIRCWmb4yJBgyiugXLNMkLA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-user-agent-node": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.587.0.tgz", - "integrity": "sha512-Pnl+DUe/bvnbEEDHP3iVJrOtE3HbFJBPgsD6vJ+ml/+IYk1Eq49jEG+EHZdNTPz3SDG0kbp2+7u41MKYJHR/iQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/config-resolver": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-3.0.1.tgz", - "integrity": "sha512-hbkYJc20SBDz2qqLzttjI/EqXemtmWk0ooRznLsiXp3066KQRTvuKHa7U4jCZCJq6Dozqvy0R1/vNESC9inPJg==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/credential-provider-imds": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-3.1.0.tgz", - "integrity": "sha512-q4A4d38v8pYYmseu/jTS3Z5I3zXlEOe5Obi+EJreVKgSVyWUHOd7/yaVCinC60QG4MRyCs98tcxBH1IMC0bu7Q==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/hash-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-3.0.0.tgz", - "integrity": "sha512-84qXstNemP3XS5jcof0el6+bDfjzuvhJPQTEfro3lgtbCtKgzPm3MgiS6ehXVPjeQ5+JS0HqmTz8f/RYfzHVxw==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/invalid-dependency": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-3.0.0.tgz", - "integrity": "sha512-F6wBBaEFgJzj0s4KUlliIGPmqXemwP6EavgvDqYwCH40O5Xr2iMHvS8todmGVZtuJCorBkXsYLyTu4PuizVq5g==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-content-length": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-3.0.0.tgz", - "integrity": "sha512-3C4s4d/iGobgCtk2tnWW6+zSTOBg1PRAm2vtWZLdriwTroFbbWNSr3lcyzHdrQHnEXYCC5K52EbpfodaIUY8sg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "requires": { - "@smithy/types": "^3.0.0" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-body-length-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-3.0.0.tgz", - "integrity": "sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-body-length-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-3.0.0.tgz", - "integrity": "sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-defaults-mode-browser": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-3.0.3.tgz", - "integrity": "sha512-3DFON2bvXJAukJe+qFgPV/rorG7ZD3m4gjCXHD1V5z/tgKQp5MCTCLntrd686tX6tj8Uli3lefWXJudNg5WmCA==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-defaults-mode-node": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-3.0.3.tgz", - "integrity": "sha512-D0b8GJXecT00baoSQ3Iieu3k3mZ7GY8w1zmg8pdogYrGvWJeLcIclqk2gbkG4K0DaBGWrO6v6r20iwIFfDYrmA==", - "requires": { - "@smithy/config-resolver": "^3.0.1", - "@smithy/credential-provider-imds": "^3.1.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/property-provider": "^3.1.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "requires": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "@aws-sdk/client-sts": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-sts/-/client-sts-3.427.0.tgz", - "integrity": "sha512-le2wLJKILyWuRfPz2HbyaNtu5kEki+ojUkTqCU6FPDRrqUvEkaaCBH9Awo/2AtrCfRkiobop8RuTTj6cAnpiJg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-sdk-sts": "3.425.0", - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/region-config-resolver": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/protocol-http": "^3.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "fast-xml-parser": "4.2.5", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/core": { - "version": "3.588.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.588.0.tgz", - "integrity": "sha512-O1c2+9ce46Z+iiid+W3iC1IvPbfIo5ev9CBi54GdNB9SaI8/3+f8MJcux0D6c9toCF0ArMersN/gp8ek57e9uQ==", - "requires": { - "@smithy/core": "^2.1.1", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "fast-xml-parser": "4.2.5", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/credential-provider-cognito-identity": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-cognito-identity/-/credential-provider-cognito-identity-3.427.0.tgz", - "integrity": "sha512-BQNzNrMJlBAfXhYNdAUqaVASpT9Aho5swj7glZKxx4Uds1w5Pih2e14JWgnl8XgUWAZ36pchTrV1aA4JT7N8vw==", - "requires": { - "@aws-sdk/client-cognito-identity": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-env": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.425.0.tgz", - "integrity": "sha512-J20etnLvMKXRVi5FK4F8yOCNm2RTaQn5psQTGdDEPWJNGxohcSpzzls8U2KcMyUJ+vItlrThr4qwgpHG3i/N0w==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-http": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.425.0.tgz", - "integrity": "sha512-aP9nkoVWf+OlNMecrUqe4+RuQrX13nucVbty0HTvuwfwJJj0T6ByWZzle+fo1D+5OxvJmtzTflBWt6jUERdHWA==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-ini": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.427.0.tgz", - "integrity": "sha512-NmH1cO/w98CKMltYec3IrJIIco19wRjATFNiw83c+FGXZ+InJwReqBnruxIOmKTx2KDzd6fwU1HOewS7UjaaaQ==", - "requires": { - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-node": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.427.0.tgz", - "integrity": "sha512-wYYbQ57nKL8OfgRbl8k6uXcdnYml+p3LSSfDUAuUEp1HKlQ8lOXFJ3BdLr5qrk7LhpyppSRnWBmh2c3kWa7ANQ==", - "requires": { - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-ini": "3.427.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-process": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.425.0.tgz", - "integrity": "sha512-YY6tkLdvtb1Fgofp3b1UWO+5vwS14LJ/smGmuGpSba0V7gFJRdcrJ9bcb9vVgAGuMdjzRJ+bUKlLLtqXkaykEw==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-sso": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.427.0.tgz", - "integrity": "sha512-c+tXyS/i49erHs4bAp6vKNYeYlyQ0VNMBgoco0LCn1rL0REtHbfhWMnqDLF6c2n3yIWDOTrQu0D73Idnpy16eA==", - "requires": { - "@aws-sdk/client-sso": "3.427.0", - "@aws-sdk/token-providers": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-provider-web-identity": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.425.0.tgz", - "integrity": "sha512-/0R65TgRzL01JU3SzloivWNwdkbIhr06uY/F5pBHf/DynQqaspKNfdHn6AiozgSVDfwRHFjKBTUy6wvf3QFkuA==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/credential-providers": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-providers/-/credential-providers-3.427.0.tgz", - "integrity": "sha512-rKKohSHju462vo+uQnPjcEZPBAfAMgGH6K1XyyCNpuOC0yYLkG87PYpvAQeb8riTrkHPX0dYUHuTHZ6zQgMGjA==", - "requires": { - "@aws-sdk/client-cognito-identity": "3.427.0", - "@aws-sdk/client-sso": "3.427.0", - "@aws-sdk/client-sts": "3.427.0", - "@aws-sdk/credential-provider-cognito-identity": "3.427.0", - "@aws-sdk/credential-provider-env": "3.425.0", - "@aws-sdk/credential-provider-http": "3.425.0", - "@aws-sdk/credential-provider-ini": "3.427.0", - "@aws-sdk/credential-provider-node": "3.427.0", - "@aws-sdk/credential-provider-process": "3.425.0", - "@aws-sdk/credential-provider-sso": "3.427.0", - "@aws-sdk/credential-provider-web-identity": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/credential-provider-imds": "^2.0.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/lib-storage": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/lib-storage/-/lib-storage-3.427.0.tgz", - "integrity": "sha512-JE26Zo4SMMY2SGlD/FilF5MpLuP8D2IrW+ye/J77dwh91gyGnNa/lubJ7WbVjIAxgeaDHsAkcpNO4VR5k6JCKg==", - "requires": { - "@smithy/abort-controller": "^2.0.1", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/smithy-client": "^2.1.9", - "buffer": "5.6.0", - "events": "3.3.0", - "stream-browserify": "3.0.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-bucket-endpoint": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.587.0.tgz", - "integrity": "sha512-HkFXLPl8pr6BH/Q0JpOESqEKL0ZK3sk7aSZ1S6GE4RXET7H5R94THULXqQFZzD48gZcyFooO/yNKZTqrZFaWKg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-arn-parser": "3.568.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-expect-continue": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.577.0.tgz", - "integrity": "sha512-6dPp8Tv4F0of4un5IAyG6q++GrRrNQQ4P2NAMB1W0VO4JoEu1C8GievbbDLi88TFIFmtKpnHB0ODCzwnoe8JsA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-flexible-checksums": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.587.0.tgz", - "integrity": "sha512-URMwp/budDvKhIvZ4a6zIBfFTun/iDlPWXqsGKYjEtHt8jz27OSjCZtDtIeqW4WTBdKL8KZgQcl+DdaE5M1qiQ==", - "requires": { - "@aws-crypto/crc32": "3.0.0", - "@aws-crypto/crc32c": "3.0.0", - "@aws-sdk/types": "3.577.0", - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-host-header": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-host-header/-/middleware-host-header-3.425.0.tgz", - "integrity": "sha512-E5Gt41LObQ+cr8QnLthwsH3MtVSNXy1AKJMowDr85h0vzqA/FHUkgHyOGntgozzjXT5M0MaSRYxS0xwTR5D4Ew==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-location-constraint": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.577.0.tgz", - "integrity": "sha512-DKPTD2D2s+t2QUo/IXYtVa/6Un8GZ+phSTBkyBNx2kfZz4Kwavhl/JJzSqTV3GfCXkVdFu7CrjoX7BZ6qWeTUA==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-logger": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-logger/-/middleware-logger-3.425.0.tgz", - "integrity": "sha512-INE9XWRXx2f4a/r2vOU0tAmgctVp7nEaEasemNtVBYhqbKLZvr9ndLBSgKGgJ8LIcXAoISipaMuFiqIGkFsm7A==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-recursion-detection": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.425.0.tgz", - "integrity": "sha512-77gnzJ5b91bgD75L/ugpOyerx6lR3oyS4080X1YI58EzdyBMkDrHM4FbMcY2RynETi3lwXCFzLRyZjWXY1mRlw==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-sdk-s3": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.587.0.tgz", - "integrity": "sha512-vtXTGEiw1E9Fax4LmcU2Z208gbrC8ShrdsSLmGcRPpu5NPOGBFBSDG5sy5EDNClrFxIl/Le8coQnD0EDBtx+uQ==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-arn-parser": "3.568.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-config-provider": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-config-provider": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-3.0.0.tgz", - "integrity": "sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-sdk-sts": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.425.0.tgz", - "integrity": "sha512-JFojrg76oKAoBknnr9EL5N2aJ1mRCtBqXoZYST58GSx8uYdFQ89qS65VNQ8JviBXzsrCNAn4vDhZ5Ch5E6TxGQ==", - "requires": { - "@aws-sdk/middleware-signing": "3.425.0", - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-signing": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-signing/-/middleware-signing-3.425.0.tgz", - "integrity": "sha512-ZpOfgJHk7ovQ0sSwg3tU4NxFOnz53lJlkJRf7S+wxQALHM0P2MJ6LYBrZaFMVsKiJxNIdZBXD6jclgHg72ZW6Q==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/signature-v4": "^2.0.0", - "@smithy/types": "^2.3.4", - "@smithy/util-middleware": "^2.0.3", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/middleware-ssec": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-ssec/-/middleware-ssec-3.577.0.tgz", - "integrity": "sha512-i2BPJR+rp8xmRVIGc0h1kDRFcM2J9GnClqqpc+NLSjmYadlcg4mPklisz9HzwFVcRPJ5XcGf3U4BYs5G8+iTyg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/middleware-user-agent": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.427.0.tgz", - "integrity": "sha512-y9HxYsNvnA3KqDl8w1jHeCwz4P9CuBEtu/G+KYffLeAMBsMZmh4SIkFFCO9wE/dyYg6+yo07rYcnnIfy7WA0bw==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/region-config-resolver": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/region-config-resolver/-/region-config-resolver-3.425.0.tgz", - "integrity": "sha512-u7uv/iUOapIJdRgRkO3wnpYsUgV6ponsZJQgVg/8L+n+Vo5PQL5gAcIuAOwcYSKQPFaeK+KbmByI4SyOK203Vw==", - "requires": { - "@smithy/node-config-provider": "^2.0.13", - "@smithy/types": "^2.3.4", - "@smithy/util-config-provider": "^2.0.0", - "@smithy/util-middleware": "^2.0.3", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/s3-request-presigner": { - "version": "3.590.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/s3-request-presigner/-/s3-request-presigner-3.590.0.tgz", - "integrity": "sha512-bb8NEG2IUHqFQJsLzr1nlkTZYyokeo3bGbHwMBKZHbdF+OXrQx0kQUcaDCXYWmeydSfHXxweQEJ2U5i1YEvT/A==", - "requires": { - "@aws-sdk/signature-v4-multi-region": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@aws-sdk/util-format-url": "3.577.0", - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/signature-v4-multi-region": { - "version": "3.587.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.587.0.tgz", - "integrity": "sha512-TR9+ZSjdXvXUz54ayHcCihhcvxI9W7102J1OK6MrLgBlPE7uRhAx42BR9L5lLJ86Xj3LuqPWf//o9d/zR9WVIg==", - "requires": { - "@aws-sdk/middleware-sdk-s3": "3.587.0", - "@aws-sdk/types": "3.577.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/signature-v4": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/signature-v4": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-3.0.0.tgz", - "integrity": "sha512-kXFOkNX+BQHe2qnLxpMEaCRGap9J6tUGLzc3A9jdn+nD4JdMwCKTJ+zFwQ20GkY+mAXGatyTw3HcoUlR39HwmA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/token-providers": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.427.0.tgz", - "integrity": "sha512-4E5E+4p8lJ69PBY400dJXF06LUHYx5lkKzBEsYqWWhoZcoftrvi24ltIhUDoGVLkrLcTHZIWSdFAWSos4hXqeg==", - "requires": { - "@aws-crypto/sha256-browser": "3.0.0", - "@aws-crypto/sha256-js": "3.0.0", - "@aws-sdk/middleware-host-header": "3.425.0", - "@aws-sdk/middleware-logger": "3.425.0", - "@aws-sdk/middleware-recursion-detection": "3.425.0", - "@aws-sdk/middleware-user-agent": "3.427.0", - "@aws-sdk/types": "3.425.0", - "@aws-sdk/util-endpoints": "3.427.0", - "@aws-sdk/util-user-agent-browser": "3.425.0", - "@aws-sdk/util-user-agent-node": "3.425.0", - "@smithy/config-resolver": "^2.0.11", - "@smithy/fetch-http-handler": "^2.2.1", - "@smithy/hash-node": "^2.0.10", - "@smithy/invalid-dependency": "^2.0.10", - "@smithy/middleware-content-length": "^2.0.12", - "@smithy/middleware-endpoint": "^2.0.10", - "@smithy/middleware-retry": "^2.0.13", - "@smithy/middleware-serde": "^2.0.10", - "@smithy/middleware-stack": "^2.0.4", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/node-http-handler": "^2.1.6", - "@smithy/property-provider": "^2.0.0", - "@smithy/protocol-http": "^3.0.6", - "@smithy/shared-ini-file-loader": "^2.0.6", - "@smithy/smithy-client": "^2.1.9", - "@smithy/types": "^2.3.4", - "@smithy/url-parser": "^2.0.10", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-body-length-browser": "^2.0.0", - "@smithy/util-body-length-node": "^2.1.0", - "@smithy/util-defaults-mode-browser": "^2.0.13", - "@smithy/util-defaults-mode-node": "^2.0.15", - "@smithy/util-retry": "^2.0.3", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/types": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.425.0.tgz", - "integrity": "sha512-6lqbmorwerN4v+J5dqbHPAsjynI0mkEF+blf+69QTaKKGaxBBVaXgqoqul9RXYcK5MMrrYRbQIMd0zYOoy90kA==", - "requires": { - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-arn-parser": { - "version": "3.568.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-arn-parser/-/util-arn-parser-3.568.0.tgz", - "integrity": "sha512-XUKJWWo+KOB7fbnPP0+g/o5Ulku/X53t7i/h+sPHr5xxYTJJ9CYnbToo95mzxe7xWvkLrsNtJ8L+MnNn9INs2w==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@aws-sdk/util-endpoints": { - "version": "3.427.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-endpoints/-/util-endpoints-3.427.0.tgz", - "integrity": "sha512-rSyiAIFF/EVvity/+LWUqoTMJ0a25RAc9iqx0WZ4tf1UjuEXRRXxZEb+jEZg1bk+pY84gdLdx9z5E+MSJCZxNQ==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/node-config-provider": "^2.0.13", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-format-url": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-format-url/-/util-format-url-3.577.0.tgz", - "integrity": "sha512-SyEGC2J+y/krFRuPgiF02FmMYhqbiIkOjDE6k4nYLJQRyS6XEAGxZoG+OHeOVEM+bsDgbxokXZiM3XKGu6qFIg==", - "requires": { - "@aws-sdk/types": "3.577.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@aws-sdk/types": { - "version": "3.577.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.577.0.tgz", - "integrity": "sha512-FT2JZES3wBKN/alfmhlo+3ZOq/XJ0C7QOZcDNrpKjB0kqYoKjhVKZ/Hx6ArR0czkKfHzBBEs6y40ebIHx2nSmA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@aws-sdk/util-locate-window": { - "version": "3.310.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.310.0.tgz", - "integrity": "sha512-qo2t/vBTnoXpjKxlsC2e1gBrRm80M3bId27r0BRB2VniSSe7bL1mmzM+/HFtujm0iAxtPM+aLEflLJlJeDPg0w==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-user-agent-browser": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.425.0.tgz", - "integrity": "sha512-22Y9iMtjGcFjGILR6/xdp1qRezlHVLyXtnpEsbuPTiernRCPk6zfAnK/ATH77r02MUjU057tdxVkd5umUBTn9Q==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/types": "^2.3.4", - "bowser": "^2.11.0", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-user-agent-node": { - "version": "3.425.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.425.0.tgz", - "integrity": "sha512-SIR4F5uQeeVAi8lv4OgRirtdtNi5zeyogTuQgGi9su8F/WP1N6JqxofcwpUY5f8/oJ2UlXr/tx1f09UHfJJzvA==", - "requires": { - "@aws-sdk/types": "3.425.0", - "@smithy/node-config-provider": "^2.0.13", - "@smithy/types": "^2.3.4", - "tslib": "^2.5.0" - } - }, - "@aws-sdk/util-utf8-browser": { - "version": "3.259.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz", - "integrity": "sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw==", - "requires": { - "tslib": "^2.3.1" - } - }, - "@aws-sdk/xml-builder": { - "version": "3.575.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.575.0.tgz", - "integrity": "sha512-cWgAwmbFYNCFzPwxL705+lWps0F3ZvOckufd2KKoEZUmtpVw9/txUXNrPySUXSmRTSRhoatIMABNfStWR043bQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/abort-controller": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", - "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", - "requires": { - "tslib": "^2.2.0" - } - }, - "@azure/core-auth": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.7.2.tgz", - "integrity": "sha512-Igm/S3fDYmnMq1uKS38Ae1/m37B3zigdlZw+kocwEhh5GjyKjPrXKO2J6rzpC1wAxrNil/jX9BJRqBshyjnF3g==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-util": "^1.1.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-client": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.2.tgz", - "integrity": "sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-rest-pipeline": "^1.9.1", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.6.1", - "@azure/logger": "^1.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-http-compat": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.1.2.tgz", - "integrity": "sha512-5MnV1yqzZwgNLLjlizsU3QqOeQChkIXw781Fwh1xdAqJR5AA32IUaq6xv1BICJvfbHoa+JYcaij2HFkhLbNTJQ==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-client": "^1.3.0", - "@azure/core-rest-pipeline": "^1.3.0" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-lro": { - "version": "2.5.4", - "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.5.4.tgz", - "integrity": "sha512-3GJiMVH7/10bulzOKGrrLeG/uCBH/9VtxqaMcB9lIqAeamI/xYQSHJL/KcsLDuH+yTjYpro/u6D/MuRe4dN70Q==", - "requires": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-util": "^1.2.0", - "@azure/logger": "^1.0.0", - "tslib": "^2.2.0" - } - }, - "@azure/core-paging": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.5.0.tgz", - "integrity": "sha512-zqWdVIt+2Z+3wqxEOGzR5hXFZ8MGKK52x4vFLw8n58pR6ZfKRx3EXYTxTaYxYHc/PexPUTyimcTWFJbji9Z6Iw==", - "requires": { - "tslib": "^2.2.0" - } - }, - "@azure/core-rest-pipeline": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.16.0.tgz", - "integrity": "sha512-CeuTvsXxCUmEuxH5g/aceuSl6w2EugvNHKAtKKVdiX915EjJJxAwfzNNWZreNnbxHZ2fi0zaM6wwS23x2JVqSQ==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-tracing": "^1.0.1", - "@azure/core-util": "^1.9.0", - "@azure/logger": "^1.0.0", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-tracing": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.1.2.tgz", - "integrity": "sha512-dawW9ifvWAWmUm9/h+/UQ2jrdvjCJ7VJEuCJ6XVNudzcOwm53BFZH4Q845vjfgoUAM8ZxokvVNxNxAITc502YA==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@azure/core-util": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.9.0.tgz", - "integrity": "sha512-AfalUQ1ZppaKuxPPMsFEUdX6GZPB3d9paR9d/TTL7Ow2De8cJaC7ibi7kWVlFAVPCYo31OcnGymc0R89DX8Oaw==", - "requires": { - "@azure/abort-controller": "^2.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@azure/abort-controller": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", - "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@azure/core-xml": { - "version": "1.4.2", - "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.4.2.tgz", - "integrity": "sha512-CW3MZhApe/S4iikbYKE7s83fjDBPIr2kpidX+hlGRwh7N4o1nIpQ/PfJTeioqhfqdMvRtheEl+ft64fyTaLNaA==", - "requires": { - "fast-xml-parser": "^4.3.2", - "tslib": "^2.6.2" - }, - "dependencies": { - "fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", - "requires": { - "strnum": "^1.0.5" - } - } - } - }, - "@azure/logger": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.0.4.tgz", - "integrity": "sha512-ustrPY8MryhloQj7OWGe+HrYx+aoiOxzbXTtgblbV3xwCqpzUK36phH3XNHQKj3EPonyFUuDTfR3qFhTEAuZEg==", - "requires": { - "tslib": "^2.2.0" - } - }, - "@azure/storage-blob": { - "version": "12.23.0", - "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.23.0.tgz", - "integrity": "sha512-c1KJ5R5hqR/HtvmFtTn/Y1BNMq45NUBp0LZH7yF8WFMET+wmESgEr0FVTu/Z5NonmfUjbgJZG5Nh8xHc5RdWGQ==", - "requires": { - "@azure/abort-controller": "^1.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-client": "^1.6.2", - "@azure/core-http-compat": "^2.0.0", - "@azure/core-lro": "^2.2.0", - "@azure/core-paging": "^1.1.1", - "@azure/core-rest-pipeline": "^1.10.1", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.6.1", - "@azure/core-xml": "^1.3.2", - "@azure/logger": "^1.0.0", - "events": "^3.0.0", - "tslib": "^2.2.0" - } - }, - "@babel/code-frame": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", - "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", - "dev": true, - "requires": { - "@babel/highlight": "^7.22.13", - "chalk": "^2.4.2" - }, - "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } - } - }, - "@babel/compat-data": { - "version": "7.21.0", - "dev": true - }, - "@babel/core": { - "version": "7.21.3", - "dev": true, - "requires": { - "@ampproject/remapping": "^2.2.0", - "@babel/code-frame": "^7.18.6", - "@babel/generator": "^7.21.3", - "@babel/helper-compilation-targets": "^7.20.7", - "@babel/helper-module-transforms": "^7.21.2", - "@babel/helpers": "^7.21.0", - "@babel/parser": "^7.21.3", - "@babel/template": "^7.20.7", - "@babel/traverse": "7.23.2", - "@babel/types": "^7.21.3", - "convert-source-map": "^1.7.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.2", - "semver": "^7.5.3" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "@babel/generator": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.3.tgz", - "integrity": "sha512-keeZWAV4LU3tW0qRi19HRpabC/ilM0HRBBzf9/k8FFiG4KVpiv0FIy4hHfLfFQZNhziCTPTmd59zoyv6DNISzg==", - "dev": true, - "requires": { - "@babel/types": "^7.23.3", - "@jridgewell/gen-mapping": "^0.3.2", - "@jridgewell/trace-mapping": "^0.3.17", - "jsesc": "^2.5.1" - }, - "dependencies": { - "@jridgewell/gen-mapping": { - "version": "0.3.2", - "dev": true, - "requires": { - "@jridgewell/set-array": "^1.0.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" - } - } - } - }, - "@babel/helper-compilation-targets": { - "version": "7.20.7", - "dev": true, - "requires": { - "@babel/compat-data": "^7.20.5", - "@babel/helper-validator-option": "^7.18.6", - "browserslist": "^4.21.3", - "lru-cache": "^5.1.1", - "semver": "^7.5.3" - } - }, - "@babel/helper-environment-visitor": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", - "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", - "dev": true - }, - "@babel/helper-function-name": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", - "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", - "dev": true, - "requires": { - "@babel/template": "^7.22.15", - "@babel/types": "^7.23.0" - } - }, - "@babel/helper-hoist-variables": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", - "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", - "dev": true, - "requires": { - "@babel/types": "^7.22.5" - } - }, - "@babel/helper-module-imports": { - "version": "7.18.6", - "dev": true, - "requires": { - "@babel/types": "^7.18.6" - } - }, - "@babel/helper-module-transforms": { - "version": "7.21.2", - "dev": true, - "requires": { - "@babel/helper-environment-visitor": "^7.18.9", - "@babel/helper-module-imports": "^7.18.6", - "@babel/helper-simple-access": "^7.20.2", - "@babel/helper-split-export-declaration": "^7.18.6", - "@babel/helper-validator-identifier": "^7.19.1", - "@babel/template": "^7.20.7", - "@babel/traverse": "7.23.2", - "@babel/types": "^7.21.2" - } - }, - "@babel/helper-simple-access": { - "version": "7.20.2", - "dev": true, - "requires": { - "@babel/types": "^7.20.2" - } - }, - "@babel/helper-split-export-declaration": { - "version": "7.22.6", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", - "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", - "dev": true, - "requires": { - "@babel/types": "^7.22.5" - } - }, - "@babel/helper-string-parser": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", - "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", - "dev": true - }, - "@babel/helper-validator-identifier": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", - "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", - "dev": true - }, - "@babel/helper-validator-option": { - "version": "7.21.0", - "dev": true - }, - "@babel/helpers": { - "version": "7.21.0", - "dev": true, - "requires": { - "@babel/template": "^7.20.7", - "@babel/traverse": "7.23.2", - "@babel/types": "^7.21.0" - } - }, - "@babel/highlight": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", - "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", - "dev": true, - "requires": { - "@babel/helper-validator-identifier": "^7.22.20", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0" - }, - "dependencies": { - "ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "requires": { - "color-convert": "^1.9.0" - } - }, - "chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "requires": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - } - }, - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true - }, - "has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } - } - }, - "@babel/parser": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.3.tgz", - "integrity": "sha512-uVsWNvlVsIninV2prNz/3lHCb+5CJ+e+IUBfbjToAHODtfGYLfCFuY4AU7TskI+dAKk+njsPiBjq1gKTvZOBaw==", - "dev": true - }, - "@babel/template": { - "version": "7.22.15", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", - "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.22.13", - "@babel/parser": "^7.22.15", - "@babel/types": "^7.22.15" - } - }, - "@babel/traverse": { - "version": "7.23.2", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.2.tgz", - "integrity": "sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw==", - "dev": true, - "requires": { - "@babel/code-frame": "^7.22.13", - "@babel/generator": "^7.23.0", - "@babel/helper-environment-visitor": "^7.22.20", - "@babel/helper-function-name": "^7.23.0", - "@babel/helper-hoist-variables": "^7.22.5", - "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/parser": "^7.23.0", - "@babel/types": "^7.23.0", - "debug": "^4.1.0", - "globals": "^11.1.0" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "@babel/types": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.3.tgz", - "integrity": "sha512-OZnvoH2l8PK5eUvEcUyCt/sXgr/h+UWpVuBbOljwcrAgUl6lpchoQ++PHGyQy1AtYnVA6CEq3y5xeEI10brpXw==", - "dev": true, - "requires": { - "@babel/helper-string-parser": "^7.22.5", - "@babel/helper-validator-identifier": "^7.22.20", - "to-fast-properties": "^2.0.0" - } - }, - "@colors/colors": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", - "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==" - }, - "@cspotcode/source-map-support": { - "version": "0.8.1", - "dev": true, - "requires": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "dependencies": { - "@jridgewell/trace-mapping": { - "version": "0.3.9", - "dev": true, - "requires": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - } - } - }, - "@dabh/diagnostics": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", - "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", - "requires": { - "colorspace": "1.1.x", - "enabled": "2.0.x", - "kuler": "^2.0.0" - } - }, - "@eslint-community/eslint-utils": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", - "integrity": "sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==", - "dev": true, - "requires": { - "eslint-visitor-keys": "^3.3.0" - } - }, - "@eslint-community/regexpp": { - "version": "4.10.1", - "resolved": "https://registry.npmjs.org/@eslint-community/regexpp/-/regexpp-4.10.1.tgz", - "integrity": "sha512-Zm2NGpWELsQAD1xsJzGQpYfvICSsFkEpU0jxBjfdC6uNEWXcHnfs9hScFWtXVDVl+rBQJGrl4g1vcKIejpH9dA==", - "dev": true - }, - "@eslint/eslintrc": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-2.1.4.tgz", - "integrity": "sha512-269Z39MS6wVJtsoUl10L60WdkhJVdPG24Q4eZTH3nnF6lpvSShEK3wQjDX9JRWAUPvPh7COouPpU9IrqaZFvtQ==", - "dev": true, - "requires": { - "ajv": "^6.12.4", - "debug": "^4.3.2", - "espree": "^9.6.0", - "globals": "^13.19.0", - "ignore": "^5.2.0", - "import-fresh": "^3.2.1", - "js-yaml": "^4.1.0", - "minimatch": "^3.1.2", - "strip-json-comments": "^3.1.1" - }, - "dependencies": { - "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", - "dev": true, - "requires": { - "type-fest": "^0.20.2" - } - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true - } - } - }, - "@eslint/js": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-8.57.0.tgz", - "integrity": "sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==", - "dev": true - }, - "@google-cloud/paginator": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/@google-cloud/paginator/-/paginator-5.0.2.tgz", - "integrity": "sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==", - "requires": { - "arrify": "^2.0.0", - "extend": "^3.0.2" - } - }, - "@google-cloud/projectify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/projectify/-/projectify-4.0.0.tgz", - "integrity": "sha512-MmaX6HeSvyPbWGwFq7mXdo0uQZLGBYCwziiLIGq5JVX+/bdI3SAq6bP98trV5eTWfLuvsMcIC1YJOF2vfteLFA==" - }, - "@google-cloud/promisify": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@google-cloud/promisify/-/promisify-4.0.0.tgz", - "integrity": "sha512-Orxzlfb9c67A15cq2JQEyVc7wEsmFBmHjZWZYQMUyJ1qivXyMwdyNOs9odi79hze+2zqdTtu1E19IM/FtqZ10g==" - }, - "@google-cloud/storage": { - "version": "7.11.1", - "resolved": "https://registry.npmjs.org/@google-cloud/storage/-/storage-7.11.1.tgz", - "integrity": "sha512-tibLSvgw7nDohMyIelt26kBpJ59YGWA2Rzep++DFNzEzKaSuCSp56Se9iM13ZlM3j5nLzR21IBkpRN58CmvCIw==", - "requires": { - "@google-cloud/paginator": "^5.0.0", - "@google-cloud/projectify": "^4.0.0", - "@google-cloud/promisify": "^4.0.0", - "abort-controller": "^3.0.0", - "async-retry": "^1.3.3", - "duplexify": "^4.1.3", - "fast-xml-parser": "^4.3.0", - "gaxios": "^6.0.2", - "google-auth-library": "^9.6.3", - "html-entities": "^2.5.2", - "mime": "^3.0.0", - "p-limit": "^3.0.1", - "retry-request": "^7.0.0", - "teeny-request": "^9.0.0", - "uuid": "^8.0.0" - }, - "dependencies": { - "fast-xml-parser": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.4.0.tgz", - "integrity": "sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==", - "requires": { - "strnum": "^1.0.5" - } - }, - "mime": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/mime/-/mime-3.0.0.tgz", - "integrity": "sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==" - }, - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" - } - } - }, - "@humanwhocodes/config-array": { - "version": "0.11.14", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.14.tgz", - "integrity": "sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg==", - "dev": true, - "requires": { - "@humanwhocodes/object-schema": "^2.0.2", - "debug": "^4.3.1", - "minimatch": "^3.0.5" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, - "@humanwhocodes/module-importer": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz", - "integrity": "sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA==", - "dev": true - }, - "@humanwhocodes/object-schema": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@humanwhocodes/object-schema/-/object-schema-2.0.3.tgz", - "integrity": "sha512-93zYdMES/c1D69yZiKDBj0V24vqNzB/koF26KPaagAfd3P/4gUlh3Dys5ogAK+Exi9QyzlD8x/08Zt7wIKcDcA==", - "dev": true - }, - "@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "dev": true, - "requires": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "dependencies": { - "argparse": { - "version": "1.0.10", - "dev": true, - "requires": { - "sprintf-js": "~1.0.2" - } - }, - "camelcase": { - "version": "5.3.1", - "dev": true - }, - "esprima": { - "version": "4.0.1", - "dev": true - }, - "find-up": { - "version": "4.1.0", - "dev": true, - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, - "js-yaml": { - "version": "3.14.1", - "dev": true, - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - }, - "locate-path": { - "version": "5.0.0", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } - }, - "p-limit": { - "version": "2.3.0", - "dev": true, - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "dev": true, - "requires": { - "p-limit": "^2.2.0" - } - }, - "sprintf-js": { - "version": "1.0.3", - "dev": true - } - } - }, - "@istanbuljs/schema": { - "version": "0.1.3", - "dev": true - }, - "@jridgewell/gen-mapping": { - "version": "0.1.1", - "dev": true, - "requires": { - "@jridgewell/set-array": "^1.0.0", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "@jridgewell/resolve-uri": { - "version": "3.1.0", - "dev": true - }, - "@jridgewell/set-array": { - "version": "1.1.2", - "dev": true - }, - "@jridgewell/sourcemap-codec": { - "version": "1.4.14", - "dev": true - }, - "@jridgewell/trace-mapping": { - "version": "0.3.17", - "dev": true, - "requires": { - "@jridgewell/resolve-uri": "3.1.0", - "@jridgewell/sourcemap-codec": "1.4.14" - } - }, - "@napi-rs/snappy-android-arm-eabi": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm-eabi/-/snappy-android-arm-eabi-7.2.2.tgz", - "integrity": "sha512-H7DuVkPCK5BlAr1NfSU8bDEN7gYs+R78pSHhDng83QxRnCLmVIZk33ymmIwurmoA1HrdTxbkbuNl+lMvNqnytw==", - "optional": true - }, - "@napi-rs/snappy-android-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-android-arm64/-/snappy-android-arm64-7.2.2.tgz", - "integrity": "sha512-2R/A3qok+nGtpVK8oUMcrIi5OMDckGYNoBLFyli3zp8w6IArPRfg1yOfVUcHvpUDTo9T7LOS1fXgMOoC796eQw==", - "optional": true - }, - "@napi-rs/snappy-darwin-arm64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-arm64/-/snappy-darwin-arm64-7.2.2.tgz", - "integrity": "sha512-USgArHbfrmdbuq33bD5ssbkPIoT7YCXCRLmZpDS6dMDrx+iM7eD2BecNbOOo7/v1eu6TRmQ0xOzeQ6I/9FIi5g==", - "optional": true - }, - "@napi-rs/snappy-darwin-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-darwin-x64/-/snappy-darwin-x64-7.2.2.tgz", - "integrity": "sha512-0APDu8iO5iT0IJKblk2lH0VpWSl9zOZndZKnBYIc+ei1npw2L5QvuErFOTeTdHBtzvUHASB+9bvgaWnQo4PvTQ==", - "optional": true - }, - "@napi-rs/snappy-freebsd-x64": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-freebsd-x64/-/snappy-freebsd-x64-7.2.2.tgz", - "integrity": "sha512-mRTCJsuzy0o/B0Hnp9CwNB5V6cOJ4wedDTWEthsdKHSsQlO7WU9W1yP7H3Qv3Ccp/ZfMyrmG98Ad7u7lG58WXA==", - "optional": true - }, - "@napi-rs/snappy-linux-arm-gnueabihf": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm-gnueabihf/-/snappy-linux-arm-gnueabihf-7.2.2.tgz", - "integrity": "sha512-v1uzm8+6uYjasBPcFkv90VLZ+WhLzr/tnfkZ/iD9mHYiULqkqpRuC8zvc3FZaJy5wLQE9zTDkTJN1IvUcZ+Vcg==", - "optional": true - }, - "@napi-rs/snappy-linux-arm64-gnu": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-gnu/-/snappy-linux-arm64-gnu-7.2.2.tgz", - "integrity": "sha512-LrEMa5pBScs4GXWOn6ZYXfQ72IzoolZw5txqUHVGs8eK4g1HR9HTHhb2oY5ySNaKakG5sOgMsb1rwaEnjhChmQ==", - "optional": true - }, - "@napi-rs/snappy-linux-arm64-musl": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-arm64-musl/-/snappy-linux-arm64-musl-7.2.2.tgz", - "integrity": "sha512-3orWZo9hUpGQcB+3aTLW7UFDqNCQfbr0+MvV67x8nMNYj5eAeUtMmUE/HxLznHO4eZ1qSqiTwLbVx05/Socdlw==", - "optional": true - }, - "@napi-rs/snappy-linux-x64-gnu": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-gnu/-/snappy-linux-x64-gnu-7.2.2.tgz", - "integrity": "sha512-jZt8Jit/HHDcavt80zxEkDpH+R1Ic0ssiVCoueASzMXa7vwPJeF4ZxZyqUw4qeSy7n8UUExomu8G8ZbP6VKhgw==", - "optional": true - }, - "@napi-rs/snappy-linux-x64-musl": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-linux-x64-musl/-/snappy-linux-x64-musl-7.2.2.tgz", - "integrity": "sha512-Dh96IXgcZrV39a+Tej/owcd9vr5ihiZ3KRix11rr1v0MWtVb61+H1GXXlz6+Zcx9y8jM1NmOuiIuJwkV4vZ4WA==", - "optional": true - }, - "@napi-rs/snappy-win32-arm64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-arm64-msvc/-/snappy-win32-arm64-msvc-7.2.2.tgz", - "integrity": "sha512-9No0b3xGbHSWv2wtLEn3MO76Yopn1U2TdemZpCaEgOGccz1V+a/1d16Piz3ofSmnA13HGFz3h9NwZH9EOaIgYA==", - "optional": true - }, - "@napi-rs/snappy-win32-ia32-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-ia32-msvc/-/snappy-win32-ia32-msvc-7.2.2.tgz", - "integrity": "sha512-QiGe+0G86J74Qz1JcHtBwM3OYdTni1hX1PFyLRo3HhQUSpmi13Bzc1En7APn+6Pvo7gkrcy81dObGLDSxFAkQQ==", - "optional": true - }, - "@napi-rs/snappy-win32-x64-msvc": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/@napi-rs/snappy-win32-x64-msvc/-/snappy-win32-x64-msvc-7.2.2.tgz", - "integrity": "sha512-a43cyx1nK0daw6BZxVcvDEXxKMFLSBSDTAhsFD0VqSKcC7MGUBMaqyoWUcMiI7LBSz4bxUmxDWKfCYzpEmeb3w==", - "optional": true - }, - "@nodelib/fs.scandir": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", - "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, - "requires": { - "@nodelib/fs.stat": "2.0.5", - "run-parallel": "^1.1.9" - } - }, - "@nodelib/fs.stat": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", - "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true - }, - "@nodelib/fs.walk": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", - "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, - "requires": { - "@nodelib/fs.scandir": "2.1.5", - "fastq": "^1.6.0" - } - }, - "@project-sunbird/logger": { - "version": "0.0.9", - "resolved": "https://registry.npmjs.org/@project-sunbird/logger/-/logger-0.0.9.tgz", - "integrity": "sha512-5DLFvgPBa5ffeavwY7fsXQsmZ+CM9rNC3Qmbh9pa4Ygfm10fhucijG0Aqku6WN9i5EeZXLoIOhDGr2fYjKsUqg==", - "requires": { - "@types/lodash": "^4.14.149", - "@types/node": "^13.9.0", - "json-stringify-safe": "^5.0.1", - "lodash": "^4.17.15", - "winston": "^3.2.1", - "winston-daily-rotate-file": "^4.4.2" - }, - "dependencies": { - "@types/node": { - "version": "13.13.52", - "resolved": "https://registry.npmjs.org/@types/node/-/node-13.13.52.tgz", - "integrity": "sha512-s3nugnZumCC//n4moGGe6tkNMyYEdaDBitVjwPxXmR5lnMG5dHePinH2EdxkG3Rh1ghFHHixAG4NJhpJW1rthQ==" - }, - "async": { - "version": "3.2.5", - "resolved": "https://registry.npmjs.org/async/-/async-3.2.5.tgz", - "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==" - }, - "fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" - }, - "file-stream-rotator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.6.1.tgz", - "integrity": "sha512-u+dBid4PvZw17PmDeRcNOtCP9CCK/9lRN2w+r1xIS7yOL9JFrIBKTvrYsxT4P0pGtThYTn++QS5ChHaUov3+zQ==", - "requires": { - "moment": "^2.29.1" - } - }, - "logform": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.0.tgz", - "integrity": "sha512-1ulHeNPp6k/LD8H91o7VYFBng5i1BDE7HoKxVbZiGFidS1Rj65qcywLxX+pVfAPoQJEjRdvKcusKwOupHCVOVQ==", - "requires": { - "@colors/colors": "1.6.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - }, - "winston": { - "version": "3.13.0", - "resolved": "https://registry.npmjs.org/winston/-/winston-3.13.0.tgz", - "integrity": "sha512-rwidmA1w3SE4j0E5MuIufFhyJPBDG7Nu71RkZor1p2+qHvJSZ9GYDA81AyleQcZbh/+V6HjeBdfnTZJm9rSeQQ==", - "requires": { - "@colors/colors": "^1.6.0", - "@dabh/diagnostics": "^2.0.2", - "async": "^3.2.3", - "is-stream": "^2.0.0", - "logform": "^2.4.0", - "one-time": "^1.0.0", - "readable-stream": "^3.4.0", - "safe-stable-stringify": "^2.3.1", - "stack-trace": "0.0.x", - "triple-beam": "^1.3.0", - "winston-transport": "^4.7.0" - } - }, - "winston-daily-rotate-file": { - "version": "4.7.1", - "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-4.7.1.tgz", - "integrity": "sha512-7LGPiYGBPNyGHLn9z33i96zx/bd71pjBn9tqQzO3I4Tayv94WPmBNwKC7CO1wPHdP9uvu+Md/1nr6VSH9h0iaA==", - "requires": { - "file-stream-rotator": "^0.6.1", - "object-hash": "^2.0.1", - "triple-beam": "^1.3.0", - "winston-transport": "^4.4.0" - } - } - } - }, - "@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "requires": { - "type-detect": "4.0.8" - } - }, - "@sinonjs/fake-timers": { - "version": "11.2.2", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-11.2.2.tgz", - "integrity": "sha512-G2piCSxQ7oWOxwGSAyFHfPIsyeJGXYtc6mFbnFA+kRXkiEnTl8c/8jul2S329iFBnDI9HGoeWWAZvuvOkZccgw==", - "dev": true, - "requires": { - "@sinonjs/commons": "^3.0.0" - } - }, - "@sinonjs/samsam": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-8.0.0.tgz", - "integrity": "sha512-Bp8KUVlLp8ibJZrnvq2foVhP0IVX2CIprMJPK0vqGqgrDa0OHVKeZyBykqskkrdxV6yKBPmGasO8LVjAKR3Gew==", - "dev": true, - "requires": { - "@sinonjs/commons": "^2.0.0", - "lodash.get": "^4.4.2", - "type-detect": "^4.0.8" - }, - "dependencies": { - "@sinonjs/commons": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-2.0.0.tgz", - "integrity": "sha512-uLa0j859mMrg2slwQYdO/AkrOfmH+X6LTVmNTS9CqexuE2IvVORIkSpJLqePAbEnKJ77aMmCwr1NUZ57120Xcg==", - "dev": true, - "requires": { - "type-detect": "4.0.8" - } - } - } - }, - "@sinonjs/text-encoding": { - "version": "0.7.2", - "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz", - "integrity": "sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ==", - "dev": true - }, - "@smithy/abort-controller": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-2.0.11.tgz", - "integrity": "sha512-MSzE1qR2JNyb7ot3blIOT3O3H0Jn06iNDEgHRaqZUwBgx5EG+VIx24Y21tlKofzYryIOcWpIohLrIIyocD6LMA==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/chunked-blob-reader": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader/-/chunked-blob-reader-3.0.0.tgz", - "integrity": "sha512-sbnURCwjF0gSToGlsBiAmd1lRCmSn72nu9axfJu5lIx6RUEgHu6GwTMbqCdhQSi0Pumcm5vFxsi9XWXb2mTaoA==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/chunked-blob-reader-native": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-3.0.0.tgz", - "integrity": "sha512-VDkpCYW+peSuM4zJip5WDfqvg2Mo/e8yxOv3VF1m11y7B8KKMKVFtmZWDe36Fvk8rGuWrPZHHXZ7rR7uM5yWyg==", - "requires": { - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/config-resolver": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/@smithy/config-resolver/-/config-resolver-2.0.14.tgz", - "integrity": "sha512-K1K+FuWQoy8j/G7lAmK85o03O89s2Vvh6kMFmzEmiHUoQCRH1rzbDtMnGNiaMHeSeYJ6y79IyTusdRG+LuWwtg==", - "requires": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/types": "^2.3.5", - "@smithy/util-config-provider": "^2.0.0", - "@smithy/util-middleware": "^2.0.4", - "tslib": "^2.5.0" - } - }, - "@smithy/core": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-2.2.0.tgz", - "integrity": "sha512-ygLZSSKgt9bR8HAxR9mK+U5obvAJBr6zlQuhN5soYWx/amjDoQN4dTkydTypgKe6rIbUjTILyLU+W5XFwXr4kg==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-retry": "^3.0.3", - "@smithy/middleware-serde": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/fetch-http-handler": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-3.0.1.tgz", - "integrity": "sha512-uaH74i5BDj+rBwoQaXioKpI0SHBJFtOVwzrCpxZxphOW0ki5jhj7dXvDMYM2IJem8TpdFvS2iC08sjOblfFGFg==", - "requires": { - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-endpoint": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-3.0.1.tgz", - "integrity": "sha512-lQ/UOdGD4KM5kLZiAl0q8Qy3dPbynvAXKAdXnYlrA1OpaUwr+neSsVokDZpY6ZVb5Yx8jnus29uv6XWpM9P4SQ==", - "requires": { - "@smithy/middleware-serde": "^3.0.0", - "@smithy/node-config-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "@smithy/url-parser": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-retry": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-3.0.3.tgz", - "integrity": "sha512-Wve1qzJb83VEU/6q+/I0cQdAkDnuzELC6IvIBwDzUEiGpKqXgX1v10FUuZGbRS6Ov/P+HHthcAoHOJZQvZNAkA==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/service-error-classification": "^3.0.0", - "@smithy/smithy-client": "^3.1.1", - "@smithy/types": "^3.0.0", - "@smithy/util-middleware": "^3.0.0", - "@smithy/util-retry": "^3.0.0", - "tslib": "^2.6.2", - "uuid": "^9.0.1" - } - }, - "@smithy/middleware-serde": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-3.0.0.tgz", - "integrity": "sha512-I1vKG1foI+oPgG9r7IMY1S+xBnmAn1ISqployvqkwHoSb8VPsngHDTOgYGYBonuOKndaWRUGJZrKYYLB+Ane6w==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/middleware-stack": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-3.0.0.tgz", - "integrity": "sha512-+H0jmyfAyHRFXm6wunskuNAqtj7yfmwFB6Fp37enytp2q047/Od9xetEaUbluyImOlGnGpaVGaVfjwawSr+i6Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/node-http-handler": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-3.0.0.tgz", - "integrity": "sha512-3trD4r7NOMygwLbUJo4eodyQuypAWr7uvPnebNJ9a70dQhVn+US8j/lCnvoJS6BXfZeF7PkkkI0DemVJw+n+eQ==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/querystring-builder": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/protocol-http": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-4.0.0.tgz", - "integrity": "sha512-qOQZOEI2XLWRWBO9AgIYuHuqjZ2csyr8/IlgFDHDNuIgLAMRx2Bl8ck5U5D6Vh9DPdoaVpuzwWMa0xcdL4O/AQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-builder": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-3.0.0.tgz", - "integrity": "sha512-bW8Fi0NzyfkE0TmQphDXr1AmBDbK01cA4C1Z7ggwMAU5RDz5AAv/KmoRwzQAS0kxXNf/D2ALTEgwK0U2c4LtRg==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-uri-escape": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/querystring-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-3.0.0.tgz", - "integrity": "sha512-UzHwthk0UEccV4dHzPySnBy34AWw3V9lIqUTxmozQ+wPDAO9csCWMfOLe7V9A2agNYy7xE+Pb0S6K/J23JSzfQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/service-error-classification": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-3.0.0.tgz", - "integrity": "sha512-3BsBtOUt2Gsnc3X23ew+r2M71WwtpHfEDGhHYHSDg6q1t8FrWh15jT25DLajFV1H+PpxAJ6gqe9yYeRUsmSdFA==", - "requires": { - "@smithy/types": "^3.0.0" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/smithy-client": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-3.1.1.tgz", - "integrity": "sha512-tj4Ku7MpzZR8cmVuPcSbrLFVxmptWktmJMwST/uIEq4sarabEdF8CbmQdYB7uJ/X51Qq2EYwnRsoS7hdR4B7rA==", - "requires": { - "@smithy/middleware-endpoint": "^3.0.1", - "@smithy/middleware-stack": "^3.0.0", - "@smithy/protocol-http": "^4.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-stream": "^3.0.1", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/url-parser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-3.0.0.tgz", - "integrity": "sha512-2XLazFgUu+YOGHtWihB3FSLAfCUajVfNBXGGYjOaVKjLAuAxx3pSBY3hBgLzIgB17haf59gOG3imKqTy8mcrjw==", - "requires": { - "@smithy/querystring-parser": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-base64": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-3.0.0.tgz", - "integrity": "sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-middleware": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-3.0.0.tgz", - "integrity": "sha512-q5ITdOnV2pXHSVDnKWrwgSNTDBAMHLptFE07ua/5Ty5WJ11bvr0vk2a7agu7qRhrCFRQlno5u3CneU5EELK+DQ==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-retry": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-3.0.0.tgz", - "integrity": "sha512-nK99bvJiziGv/UOKJlDvFF45F00WgPLKVIGUfAK+mDhzVN2hb/S33uW2Tlhg5PVBoqY7tDVqL0zmu4OxAHgo9g==", - "requires": { - "@smithy/service-error-classification": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-stream": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-3.0.1.tgz", - "integrity": "sha512-7F7VNNhAsfMRA8I986YdOY5fE0/T1/ZjFF6OLsqkvQVNP3vZ/szYDfGCyphb7ioA09r32K/0qbSFfNFU68aSzA==", - "requires": { - "@smithy/fetch-http-handler": "^3.0.1", - "@smithy/node-http-handler": "^3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-base64": "^3.0.0", - "@smithy/util-buffer-from": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-uri-escape": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-3.0.0.tgz", - "integrity": "sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "@smithy/credential-provider-imds": { - "version": "2.0.16", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-2.0.16.tgz", - "integrity": "sha512-tKa2xF+69TvGxJT+lnJpGrKxUuAZDLYXFhqnPEgnHz+psTpkpcB4QRjHj63+uj83KaeFJdTfW201eLZeRn6FfA==", - "requires": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/property-provider": "^2.0.12", - "@smithy/types": "^2.3.5", - "@smithy/url-parser": "^2.0.11", - "tslib": "^2.5.0" - } - }, - "@smithy/eventstream-codec": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-2.0.11.tgz", - "integrity": "sha512-BQCTjxhCYRZIfXapa2LmZSaH8QUBGwMZw7XRN83hrdixbLjIcj+o549zjkedFS07Ve2TlvWUI6BTzP+nv7snBA==", - "requires": { - "@aws-crypto/crc32": "3.0.0", - "@smithy/types": "^2.3.5", - "@smithy/util-hex-encoding": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/eventstream-serde-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-3.0.0.tgz", - "integrity": "sha512-NB7AFiPN4NxP/YCAnrvYR18z2/ZsiHiF7VtG30gshO9GbFrIb1rC8ep4NGpJSWrz6P64uhPXeo4M0UsCLnZKqw==", - "requires": { - "@smithy/eventstream-serde-universal": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/eventstream-serde-config-resolver": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-3.0.0.tgz", - "integrity": "sha512-RUQG3vQ3LX7peqqHAbmayhgrF5aTilPnazinaSGF1P0+tgM3vvIRWPHmlLIz2qFqB9LqFIxditxc8O2Z6psrRw==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/eventstream-serde-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-node/-/eventstream-serde-node-3.0.0.tgz", - "integrity": "sha512-baRPdMBDMBExZXIUAoPGm/hntixjt/VFpU6+VmCyiYJYzRHRxoaI1MN+5XE+hIS8AJ2GCHLMFEIOLzq9xx1EgQ==", - "requires": { - "@smithy/eventstream-serde-universal": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/eventstream-serde-universal": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-3.0.0.tgz", - "integrity": "sha512-HNFfShmotWGeAoW4ujP8meV9BZavcpmerDbPIjkJbxKbN8RsUcpRQ/2OyIxWNxXNH2GWCAxuSB7ynmIGJlQ3Dw==", - "requires": { - "@smithy/eventstream-codec": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/eventstream-codec": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/eventstream-codec/-/eventstream-codec-3.0.0.tgz", - "integrity": "sha512-PUtyEA0Oik50SaEFCZ0WPVtF9tz/teze2fDptW6WRXl+RrEenH8UbEjudOz8iakiMl3lE3lCVqYf2Y+znL8QFQ==", - "requires": { - "@aws-crypto/crc32": "3.0.0", - "@smithy/types": "^3.0.0", - "@smithy/util-hex-encoding": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-hex-encoding": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-3.0.0.tgz", - "integrity": "sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/fetch-http-handler": { - "version": "2.2.2", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-2.2.2.tgz", - "integrity": "sha512-K7aRtRuaBjzlk+jWWeyfDTLAmRRvmA4fU8eHUXtjsuEDgi3f356ZE32VD2ssxIH13RCLVZbXMt5h7wHzYiSuVA==", - "requires": { - "@smithy/protocol-http": "^3.0.7", - "@smithy/querystring-builder": "^2.0.11", - "@smithy/types": "^2.3.5", - "@smithy/util-base64": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/hash-blob-browser": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-blob-browser/-/hash-blob-browser-3.0.0.tgz", - "integrity": "sha512-/Wbpdg+bwJvW7lxR/zpWAc1/x/YkcqguuF2bAzkJrvXriZu1vm8r+PUdE4syiVwQg7PPR2dXpi3CLBb9qRDaVQ==", - "requires": { - "@smithy/chunked-blob-reader": "^3.0.0", - "@smithy/chunked-blob-reader-native": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/hash-node": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/hash-node/-/hash-node-2.0.11.tgz", - "integrity": "sha512-PbleVugN2tbhl1ZoNWVrZ1oTFFas/Hq+s6zGO8B9bv4w/StTriTKA9W+xZJACOj9X7zwfoTLbscM+avCB1KqOQ==", - "requires": { - "@smithy/types": "^2.3.5", - "@smithy/util-buffer-from": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/hash-stream-node": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/hash-stream-node/-/hash-stream-node-3.0.0.tgz", - "integrity": "sha512-J0i7de+EgXDEGITD4fxzmMX8CyCNETTIRXlxjMiNUvvu76Xn3GJ31wQR85ynlPk2wI1lqoknAFJaD1fiNDlbIA==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/invalid-dependency": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/invalid-dependency/-/invalid-dependency-2.0.11.tgz", - "integrity": "sha512-zazq99ujxYv/NOf9zh7xXbNgzoVLsqE0wle8P/1zU/XdhPi/0zohTPKWUzIxjGdqb5hkkwfBkNkl5H+LE0mvgw==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/is-array-buffer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.0.0.tgz", - "integrity": "sha512-z3PjFjMyZNI98JFRJi/U0nGoLWMSJlDjAW4QUX2WNZLas5C0CmVV6LJ01JI0k90l7FvpmixjWxPFmENSClQ7ug==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/md5-js": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/md5-js/-/md5-js-3.0.0.tgz", - "integrity": "sha512-Tm0vrrVzjlD+6RCQTx7D3Ls58S3FUH1ZCtU1MIh/qQmaOo1H9lMN2as6CikcEwgattnA9SURSdoJJ27xMcEfMA==", - "requires": { - "@smithy/types": "^3.0.0", - "@smithy/util-utf8": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/is-array-buffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-3.0.0.tgz", - "integrity": "sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - }, - "@smithy/util-buffer-from": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-3.0.0.tgz", - "integrity": "sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==", - "requires": { - "@smithy/is-array-buffer": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/util-utf8": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-3.0.0.tgz", - "integrity": "sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==", - "requires": { - "@smithy/util-buffer-from": "^3.0.0", - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/middleware-content-length": { - "version": "2.0.13", - "resolved": "https://registry.npmjs.org/@smithy/middleware-content-length/-/middleware-content-length-2.0.13.tgz", - "integrity": "sha512-Md2kxWpaec3bXp1oERFPQPBhOXCkGSAF7uc1E+4rkwjgw3/tqAXRtbjbggu67HJdwaif76As8AV6XxbD1HzqTQ==", - "requires": { - "@smithy/protocol-http": "^3.0.7", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/middleware-endpoint": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-endpoint/-/middleware-endpoint-2.0.11.tgz", - "integrity": "sha512-mCugsvB15up6fqpzUEpMT4CuJmFkEI+KcozA7QMzYguXCaIilyMKsyxgamwmr+o7lo3QdjN0//XLQ9bWFL129g==", - "requires": { - "@smithy/middleware-serde": "^2.0.11", - "@smithy/types": "^2.3.5", - "@smithy/url-parser": "^2.0.11", - "@smithy/util-middleware": "^2.0.4", - "tslib": "^2.5.0" - } - }, - "@smithy/middleware-retry": { - "version": "2.0.16", - "resolved": "https://registry.npmjs.org/@smithy/middleware-retry/-/middleware-retry-2.0.16.tgz", - "integrity": "sha512-Br5+0yoiMS0ugiOAfJxregzMMGIRCbX4PYo1kDHtLgvkA/d++aHbnHB819m5zOIAMPvPE7AThZgcsoK+WOsUTA==", - "requires": { - "@smithy/node-config-provider": "^2.1.1", - "@smithy/protocol-http": "^3.0.7", - "@smithy/service-error-classification": "^2.0.4", - "@smithy/types": "^2.3.5", - "@smithy/util-middleware": "^2.0.4", - "@smithy/util-retry": "^2.0.4", - "tslib": "^2.5.0", - "uuid": "^8.3.2" - }, - "dependencies": { - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" - } - } - }, - "@smithy/middleware-serde": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/middleware-serde/-/middleware-serde-2.0.11.tgz", - "integrity": "sha512-NuxnjMyf4zQqhwwdh0OTj5RqpnuT6HcH5Xg5GrPijPcKzc2REXVEVK4Yyk8ckj8ez1XSj/bCmJ+oNjmqB02GWA==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/middleware-stack": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/@smithy/middleware-stack/-/middleware-stack-2.0.5.tgz", - "integrity": "sha512-bVQU/rZzBY7CbSxIrDTGZYnBWKtIw+PL/cRc9B7etZk1IKSOe0NvKMJyWllfhfhrTeMF6eleCzOihIQympAvPw==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/node-config-provider": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-2.1.1.tgz", - "integrity": "sha512-1lF6s1YWBi1LBu2O30tD3jyTgMtuvk/Z1twzXM4GPYe4dmZix4nNREPJIPOcfFikNU2o0eTYP80+izx5F2jIJA==", - "requires": { - "@smithy/property-provider": "^2.0.12", - "@smithy/shared-ini-file-loader": "^2.2.0", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/node-http-handler": { - "version": "2.1.7", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-2.1.7.tgz", - "integrity": "sha512-PQIKZXlp3awCDn/xNlCSTFE7aYG/5Tx33M05NfQmWYeB5yV1GZZOSz4dXpwiNJYTXb9jPqjl+ueXXkwtEluFFA==", - "requires": { - "@smithy/abort-controller": "^2.0.11", - "@smithy/protocol-http": "^3.0.7", - "@smithy/querystring-builder": "^2.0.11", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/property-provider": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-2.0.12.tgz", - "integrity": "sha512-Un/OvvuQ1Kg8WYtoMCicfsFFuHb/TKL3pCA6ZIo/WvNTJTR94RtoRnL7mY4XkkUAoFMyf6KjcQJ76y1FX7S5rw==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/protocol-http": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/@smithy/protocol-http/-/protocol-http-3.0.7.tgz", - "integrity": "sha512-HnZW8y+r66ntYueCDbLqKwWcMNWW8o3eVpSrHNluwtBJ/EUWfQHRKSiu6vZZtc6PGfPQWgVfucoCE/C3QufMAA==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/querystring-builder": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-builder/-/querystring-builder-2.0.11.tgz", - "integrity": "sha512-b4kEbVMxpmfv2VWUITn2otckTi7GlMteZQxi+jlwedoATOGEyrCJPfRcYQJjbCi3fZ2QTfh3PcORvB27+j38Yg==", - "requires": { - "@smithy/types": "^2.3.5", - "@smithy/util-uri-escape": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/querystring-parser": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/querystring-parser/-/querystring-parser-2.0.11.tgz", - "integrity": "sha512-YXe7jhi7s3dQ0Fu9dLoY/gLu6NCyy8tBWJL/v2c9i7/RLpHgKT+uT96/OqZkHizCJ4kr0ZD46tzMjql/o60KLg==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/service-error-classification": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/service-error-classification/-/service-error-classification-2.0.4.tgz", - "integrity": "sha512-77506l12I5gxTZqBkx3Wb0RqMG81bMYLaVQ+EqIWFwQDJRs5UFeXogKxSKojCmz1wLUziHZQXm03MBzPQiumQw==", - "requires": { - "@smithy/types": "^2.3.5" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-2.2.0.tgz", - "integrity": "sha512-xFXqs4vAb5BdkzHSRrTapFoaqS4/3m/CGZzdw46fBjYZ0paYuLAoMY60ICCn1FfGirG+PiJ3eWcqJNe4/SkfyA==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/signature-v4": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-2.0.11.tgz", - "integrity": "sha512-EFVU1dT+2s8xi227l1A9O27edT/GNKvyAK6lZnIZ0zhIHq/jSLznvkk15aonGAM1kmhmZBVGpI7Tt0odueZK9A==", - "requires": { - "@smithy/eventstream-codec": "^2.0.11", - "@smithy/is-array-buffer": "^2.0.0", - "@smithy/types": "^2.3.5", - "@smithy/util-hex-encoding": "^2.0.0", - "@smithy/util-middleware": "^2.0.4", - "@smithy/util-uri-escape": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/smithy-client": { - "version": "2.1.10", - "resolved": "https://registry.npmjs.org/@smithy/smithy-client/-/smithy-client-2.1.10.tgz", - "integrity": "sha512-2OEmZDiW1Z196QHuQZ5M6cBE8FCSG0H2HADP1G+DY8P3agsvb0YJyfhyKuJbxIQy15tr3eDAK6FOrlbxgKOOew==", - "requires": { - "@smithy/middleware-stack": "^2.0.5", - "@smithy/types": "^2.3.5", - "@smithy/util-stream": "^2.0.15", - "tslib": "^2.5.0" - } - }, - "@smithy/types": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-2.3.5.tgz", - "integrity": "sha512-ehyDt8M9hehyxrLQGoA1BGPou8Js1Ocoh5M0ngDhJMqbFmNK5N6Xhr9/ZExWkyIW8XcGkiMPq3ZUEE0ScrhbuQ==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/url-parser": { - "version": "2.0.11", - "resolved": "https://registry.npmjs.org/@smithy/url-parser/-/url-parser-2.0.11.tgz", - "integrity": "sha512-h89yXMCCF+S5k9XIoKltMIWTYj+FcEkU/IIFZ6RtE222fskOTL4Iak6ZRG+ehSvZDt8yKEcxqheTDq7JvvtK3g==", - "requires": { - "@smithy/querystring-parser": "^2.0.11", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/util-base64": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-base64/-/util-base64-2.0.0.tgz", - "integrity": "sha512-Zb1E4xx+m5Lud8bbeYi5FkcMJMnn+1WUnJF3qD7rAdXpaL7UjkFQLdmW5fHadoKbdHpwH9vSR8EyTJFHJs++tA==", - "requires": { - "@smithy/util-buffer-from": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-body-length-browser": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-browser/-/util-body-length-browser-2.0.0.tgz", - "integrity": "sha512-JdDuS4ircJt+FDnaQj88TzZY3+njZ6O+D3uakS32f2VNnDo3vyEuNdBOh/oFd8Df1zSZOuH1HEChk2AOYDezZg==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-body-length-node": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/@smithy/util-body-length-node/-/util-body-length-node-2.1.0.tgz", - "integrity": "sha512-/li0/kj/y3fQ3vyzn36NTLGmUwAICb7Jbe/CsWCktW363gh1MOcpEcSO3mJ344Gv2dqz8YJCLQpb6hju/0qOWw==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-buffer-from": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.0.0.tgz", - "integrity": "sha512-/YNnLoHsR+4W4Vf2wL5lGv0ksg8Bmk3GEGxn2vEQt52AQaPSCuaO5PM5VM7lP1K9qHRKHwrPGktqVoAHKWHxzw==", - "requires": { - "@smithy/is-array-buffer": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-config-provider": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-config-provider/-/util-config-provider-2.0.0.tgz", - "integrity": "sha512-xCQ6UapcIWKxXHEU4Mcs2s7LcFQRiU3XEluM2WcCjjBtQkUN71Tb+ydGmJFPxMUrW/GWMgQEEGipLym4XG0jZg==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-defaults-mode-browser": { - "version": "2.0.14", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-2.0.14.tgz", - "integrity": "sha512-NupG7SWUucm3vJrvlpt9jG1XeoPJphjcivgcUUXhDJbUPy4F04LhlTiAhWSzwlCNcF8OJsMvZ/DWbpYD3pselw==", - "requires": { - "@smithy/property-provider": "^2.0.12", - "@smithy/smithy-client": "^2.1.10", - "@smithy/types": "^2.3.5", - "bowser": "^2.11.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-defaults-mode-node": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-2.0.18.tgz", - "integrity": "sha512-+3jMom/b/Cdp21tDnY4vKu249Al+G/P0HbRbct7/aSZDlROzv1tksaYukon6UUv7uoHn+/McqnsvqZHLlqvQ0g==", - "requires": { - "@smithy/config-resolver": "^2.0.14", - "@smithy/credential-provider-imds": "^2.0.16", - "@smithy/node-config-provider": "^2.1.1", - "@smithy/property-provider": "^2.0.12", - "@smithy/smithy-client": "^2.1.10", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/util-endpoints": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/@smithy/util-endpoints/-/util-endpoints-2.0.1.tgz", - "integrity": "sha512-ZRT0VCOnKlVohfoABMc8lWeQo/JEFuPWctfNRXgTHbyOVssMOLYFUNWukxxiHRGVAhV+n3c0kPW+zUqckjVPEA==", - "requires": { - "@smithy/node-config-provider": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/node-config-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/node-config-provider/-/node-config-provider-3.1.0.tgz", - "integrity": "sha512-ngfB8QItUfTFTfHMvKuc2g1W60V1urIgZHqD1JNFZC2tTWXahqf2XvKXqcBS7yZqR7GqkQQZy11y/lNOUWzq7Q==", - "requires": { - "@smithy/property-provider": "^3.1.0", - "@smithy/shared-ini-file-loader": "^3.1.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/property-provider": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/property-provider/-/property-provider-3.1.0.tgz", - "integrity": "sha512-Tj3+oVhqdZgemjCiWjFlADfhvLF4C/uKDuKo7/tlEsRQ9+3emCreR2xndj970QSRSsiCEU8hZW3/8JQu+n5w4Q==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/shared-ini-file-loader": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-3.1.0.tgz", - "integrity": "sha512-dAM7wSX0NR3qTNyGVN/nwwpEDzfV9T/3AN2eABExWmda5VqZKSsjlINqomO5hjQWGv+IIkoXfs3u2vGSNz8+Rg==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@smithy/util-hex-encoding": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-hex-encoding/-/util-hex-encoding-2.0.0.tgz", - "integrity": "sha512-c5xY+NUnFqG6d7HFh1IFfrm3mGl29lC+vF+geHv4ToiuJCBmIfzx6IeHLg+OgRdPFKDXIw6pvi+p3CsscaMcMA==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-middleware": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/util-middleware/-/util-middleware-2.0.4.tgz", - "integrity": "sha512-Pbu6P4MBwRcjrLgdTR1O4Y3c0sTZn2JdOiJNcgL7EcIStcQodj+6ZTXtbyU/WTEU3MV2NMA10LxFc3AWHZ3+4A==", - "requires": { - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/util-retry": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/@smithy/util-retry/-/util-retry-2.0.4.tgz", - "integrity": "sha512-b+n1jBBKc77C1E/zfBe1Zo7S9OXGBiGn55N0apfhZHxPUP/fMH5AhFUUcWaJh7NAnah284M5lGkBKuhnr3yK5w==", - "requires": { - "@smithy/service-error-classification": "^2.0.4", - "@smithy/types": "^2.3.5", - "tslib": "^2.5.0" - } - }, - "@smithy/util-stream": { - "version": "2.0.15", - "resolved": "https://registry.npmjs.org/@smithy/util-stream/-/util-stream-2.0.15.tgz", - "integrity": "sha512-A/hkYJPH2N5MCWYvky4tTpQihpYAEzqnUfxDyG3L/yMndy/2sLvxnyQal9Opuj1e9FiKSTeMyjnU9xxZGs0mRw==", - "requires": { - "@smithy/fetch-http-handler": "^2.2.2", - "@smithy/node-http-handler": "^2.1.7", - "@smithy/types": "^2.3.5", - "@smithy/util-base64": "^2.0.0", - "@smithy/util-buffer-from": "^2.0.0", - "@smithy/util-hex-encoding": "^2.0.0", - "@smithy/util-utf8": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-uri-escape": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-uri-escape/-/util-uri-escape-2.0.0.tgz", - "integrity": "sha512-ebkxsqinSdEooQduuk9CbKcI+wheijxEb3utGXkCoYQkJnwTnLbH1JXGimJtUkQwNQbsbuYwG2+aFVyZf5TLaw==", - "requires": { - "tslib": "^2.5.0" - } - }, - "@smithy/util-utf8": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.0.0.tgz", - "integrity": "sha512-rctU1VkziY84n5OXe3bPNpKR001ZCME2JCaBBFgtiM2hfKbHFudc/BkMuPab8hRbLd0j3vbnBTTZ1igBf0wgiQ==", - "requires": { - "@smithy/util-buffer-from": "^2.0.0", - "tslib": "^2.5.0" - } - }, - "@smithy/util-waiter": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/util-waiter/-/util-waiter-3.0.0.tgz", - "integrity": "sha512-+fEXJxGDLCoqRKVSmo0auGxaqbiCo+8oph+4auefYjaNxjOLKSY2MxVQfRzo65PaZv4fr+5lWg+au7vSuJJ/zw==", - "requires": { - "@smithy/abort-controller": "^3.0.0", - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - }, - "dependencies": { - "@smithy/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-p6GlFGBt9K4MYLu72YuJ523NVR4A8oHlC5M2JO6OmQqN8kAc/uh1JqLE+FizTokrSJGg0CSvC+BrsmGzKtsZKA==", - "requires": { - "@smithy/types": "^3.0.0", - "tslib": "^2.6.2" - } - }, - "@smithy/types": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-3.0.0.tgz", - "integrity": "sha512-VvWuQk2RKFuOr98gFhjca7fkBS+xLLURT8bUjk5XQoV0ZLm7WPwWPPY3/AwzTLuUBDeoKDCthfe1AsTUWaSEhw==", - "requires": { - "tslib": "^2.6.2" - } - } - } - }, - "@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==" - }, - "@tsconfig/node10": { - "version": "1.0.9", - "dev": true - }, - "@tsconfig/node12": { - "version": "1.0.11", - "dev": true - }, - "@tsconfig/node14": { - "version": "1.0.3", - "dev": true - }, - "@tsconfig/node16": { - "version": "1.0.3", - "dev": true - }, - "@types/body-parser": { - "version": "1.19.2", - "dev": true, - "requires": { - "@types/connect": "*", - "@types/node": "*" - } - }, - "@types/caseless": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/@types/caseless/-/caseless-0.12.5.tgz", - "integrity": "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg==" - }, - "@types/chai": { - "version": "4.3.4", - "dev": true - }, - "@types/chai-as-promised": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/@types/chai-as-promised/-/chai-as-promised-7.1.6.tgz", - "integrity": "sha512-cQLhk8fFarRVZAXUQV1xEnZgMoPxqKojBvRkqPCKPQCzEhpbbSKl1Uu75kDng7k5Ln6LQLUmNBjLlFthCgm1NA==", - "dev": true, - "requires": { - "@types/chai": "*" - } - }, - "@types/chai-spies": { - "version": "1.0.3", - "dev": true, - "requires": { - "@types/chai": "*" - } - }, - "@types/compression": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/@types/compression/-/compression-1.7.3.tgz", - "integrity": "sha512-rKquEGjebqizyHNMOpaE/4FdYR5VQiWFeesqYfvJU0seSEyB4625UGhNOO/qIkH10S3wftiV7oefc8WdLZ/gCQ==", - "dev": true, - "requires": { - "@types/express": "*" - } - }, - "@types/connect": { - "version": "3.4.35", - "dev": true, - "requires": { - "@types/node": "*" - } - }, - "@types/cookiejar": { - "version": "2.1.2", - "dev": true - }, - "@types/debug": { - "version": "4.1.12", - "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", - "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", - "requires": { - "@types/ms": "*" - } - }, - "@types/express": { - "version": "4.17.17", - "dev": true, - "requires": { - "@types/body-parser": "*", - "@types/express-serve-static-core": "^4.17.33", - "@types/qs": "*", - "@types/serve-static": "*" - } - }, - "@types/express-serve-static-core": { - "version": "4.17.33", - "dev": true, - "requires": { - "@types/node": "*", - "@types/qs": "*", - "@types/range-parser": "*" - } - }, - "@types/http-errors": { - "version": "2.0.1", - "dev": true - }, - "@types/kafkajs": { - "version": "1.9.0", - "dev": true, - "requires": { - "kafkajs": "*" - } - }, - "@types/knex": { - "version": "0.16.1", - "dev": true, - "requires": { - "knex": "*" - } - }, - "@types/lodash": { - "version": "4.14.191" - }, - "@types/mime": { - "version": "3.0.1", - "dev": true - }, - "@types/mocha": { - "version": "10.0.1", - "dev": true - }, - "@types/mock-knex": { - "version": "0.4.6", - "resolved": "https://registry.npmjs.org/@types/mock-knex/-/mock-knex-0.4.6.tgz", - "integrity": "sha512-7MHM9v9ZgFpAZh2NfCyYeEJKlYJSf/mVRAQ324AJZGB9fbo8dVa8u9nxibyJzwwMXYlbRXLNbKEfoK2xBJFgUw==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, - "@types/moment": { - "version": "2.13.0", - "dev": true, - "requires": { - "moment": "*" - } - }, - "@types/ms": { - "version": "0.7.34", - "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.34.tgz", - "integrity": "sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==" - }, - "@types/node": { - "version": "18.15.3" - }, - "@types/pegjs": { - "version": "0.10.6", - "resolved": "https://registry.npmjs.org/@types/pegjs/-/pegjs-0.10.6.tgz", - "integrity": "sha512-eLYXDbZWXh2uxf+w8sXS8d6KSoXTswfps6fvCUuVAGN8eRpfe7h9eSRydxiSJvo9Bf+GzifsDOr9TMQlmJdmkw==" - }, - "@types/pg": { - "version": "8.6.6", - "dev": true, - "requires": { - "@types/node": "*", - "pg-protocol": "*", - "pg-types": "^2.2.0" - } - }, - "@types/qs": { - "version": "6.9.7", - "dev": true - }, - "@types/range-parser": { - "version": "1.2.4", - "dev": true - }, - "@types/request": { - "version": "2.48.12", - "resolved": "https://registry.npmjs.org/@types/request/-/request-2.48.12.tgz", - "integrity": "sha512-G3sY+NpsA9jnwm0ixhAFQSJ3Q9JkpLZpJbI3GMv0mIAT0y3mRabYeINzal5WOChIiaTEGQYlHOKgkaM9EisWHw==", - "requires": { - "@types/caseless": "*", - "@types/node": "*", - "@types/tough-cookie": "*", - "form-data": "^2.5.0" - }, - "dependencies": { - "form-data": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.5.1.tgz", - "integrity": "sha512-m21N3WOmEEURgk6B9GLOE4RuWOFf28Lhh9qGYeNlGq4VDXUlJy2th2slBNU8Gp8EzloYZOibZJ7t5ecIrFSjVA==", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - } - } - }, - "@types/serve-static": { - "version": "1.15.1", - "dev": true, - "requires": { - "@types/mime": "*", - "@types/node": "*" - } - }, - "@types/sinon": { - "version": "17.0.3", - "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-17.0.3.tgz", - "integrity": "sha512-j3uovdn8ewky9kRBG19bOwaZbexJu/XjtkHyjvUgt4xfPFz18dcORIMqnYh66Fx3Powhcr85NT5+er3+oViapw==", - "dev": true, - "requires": { - "@types/sinonjs__fake-timers": "*" - } - }, - "@types/sinonjs__fake-timers": { - "version": "8.1.5", - "resolved": "https://registry.npmjs.org/@types/sinonjs__fake-timers/-/sinonjs__fake-timers-8.1.5.tgz", - "integrity": "sha512-mQkU2jY8jJEF7YHjHvsQO8+3ughTL1mcnn96igfhONmR+fUPSKIkefQYpSe8bsly2Ep7oQbn/6VG5/9/0qcArQ==", - "dev": true - }, - "@types/slug": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/@types/slug/-/slug-5.0.8.tgz", - "integrity": "sha512-mblTWR1OST257k1gZ3QvqG+ERSr8Ea6dyM1FH6Jtm4jeXi0/r0/95VNctofuiywPxCVQuE8AuFoqmvJ4iVUlXQ==", - "dev": true - }, - "@types/superagent": { - "version": "3.8.7", - "dev": true, - "requires": { - "@types/cookiejar": "*", - "@types/node": "*" - } - }, - "@types/tough-cookie": { - "version": "4.0.5", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.5.tgz", - "integrity": "sha512-/Ad8+nIOV7Rl++6f1BdKxFSMgmoqEoYbHRpPcx3JEfv8VRsQe9Z4mCXeJBzxs7mbHY/XOZZuXlRNfhpVPbs6ZA==" - }, - "@types/triple-beam": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.3.tgz", - "integrity": "sha512-6tOUG+nVHn0cJbVp25JFayS5UE6+xlbcNF9Lo9mU7U0zk3zeUShZied4YEQZjy1JBF043FSkdXw8YkUJuVtB5g==" - }, - "@types/uuid": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.5.tgz", - "integrity": "sha512-xfHdwa1FMJ082prjSJpoEI57GZITiQz10r3vEJCHa2khEFQjKy91aWKz6+zybzssCvXUwE1LQWgWVwZ4nYUvHQ==", - "dev": true - }, - "@types/validator": { - "version": "13.11.10", - "resolved": "https://registry.npmjs.org/@types/validator/-/validator-13.11.10.tgz", - "integrity": "sha512-e2PNXoXLr6Z+dbfx5zSh9TRlXJrELycxiaXznp4S5+D2M3b9bqJEitNHA5923jhnB2zzFiZHa2f0SI1HoIahpg==" - }, - "@typescript-eslint/eslint-plugin": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.12.0.tgz", - "integrity": "sha512-7F91fcbuDf/d3S8o21+r3ZncGIke/+eWk0EpO21LXhDfLahriZF9CGj4fbAetEjlaBdjdSm9a6VeXbpbT6Z40Q==", - "dev": true, - "requires": { - "@eslint-community/regexpp": "^4.10.0", - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/type-utils": "7.12.0", - "@typescript-eslint/utils": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "graphemer": "^1.4.0", - "ignore": "^5.3.1", - "natural-compare": "^1.4.0", - "ts-api-utils": "^1.3.0" - } - }, - "@typescript-eslint/parser": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-7.12.0.tgz", - "integrity": "sha512-dm/J2UDY3oV3TKius2OUZIFHsomQmpHtsV0FTh1WO8EKgHLQ1QCADUqscPgTpU+ih1e21FQSRjXckHn3txn6kQ==", - "dev": true, - "requires": { - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/typescript-estree": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "debug": "^4.3.4" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, - "@typescript-eslint/scope-manager": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-7.12.0.tgz", - "integrity": "sha512-itF1pTnN6F3unPak+kutH9raIkL3lhH1YRPGgt7QQOh43DQKVJXmWkpb+vpc/TiDHs6RSd9CTbDsc/Y+Ygq7kg==", - "dev": true, - "requires": { - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0" - } - }, - "@typescript-eslint/type-utils": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/type-utils/-/type-utils-7.12.0.tgz", - "integrity": "sha512-lib96tyRtMhLxwauDWUp/uW3FMhLA6D0rJ8T7HmH7x23Gk1Gwwu8UZ94NMXBvOELn6flSPiBrCKlehkiXyaqwA==", - "dev": true, - "requires": { - "@typescript-eslint/typescript-estree": "7.12.0", - "@typescript-eslint/utils": "7.12.0", - "debug": "^4.3.4", - "ts-api-utils": "^1.3.0" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, - "@typescript-eslint/types": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-7.12.0.tgz", - "integrity": "sha512-o+0Te6eWp2ppKY3mLCU+YA9pVJxhUJE15FV7kxuD9jgwIAa+w/ycGJBMrYDTpVGUM/tgpa9SeMOugSabWFq7bg==", - "dev": true - }, - "@typescript-eslint/typescript-estree": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-7.12.0.tgz", - "integrity": "sha512-5bwqLsWBULv1h6pn7cMW5dXX/Y2amRqLaKqsASVwbBHMZSnHqE/HN4vT4fE0aFsiwxYvr98kqOWh1a8ZKXalCQ==", - "dev": true, - "requires": { - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/visitor-keys": "7.12.0", - "debug": "^4.3.4", - "globby": "^11.1.0", - "is-glob": "^4.0.3", - "minimatch": "^9.0.4", - "semver": "^7.5.3", - "ts-api-utils": "^1.3.0" - }, - "dependencies": { - "brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "requires": { - "balanced-match": "^1.0.0" - } - }, - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dev": true, - "requires": { - "brace-expansion": "^2.0.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, - "@typescript-eslint/utils": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/utils/-/utils-7.12.0.tgz", - "integrity": "sha512-Y6hhwxwDx41HNpjuYswYp6gDbkiZ8Hin9Bf5aJQn1bpTs3afYY4GX+MPYxma8jtoIV2GRwTM/UJm/2uGCVv+DQ==", - "dev": true, - "requires": { - "@eslint-community/eslint-utils": "^4.4.0", - "@typescript-eslint/scope-manager": "7.12.0", - "@typescript-eslint/types": "7.12.0", - "@typescript-eslint/typescript-estree": "7.12.0" - } - }, - "@typescript-eslint/visitor-keys": { - "version": "7.12.0", - "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-7.12.0.tgz", - "integrity": "sha512-uZk7DevrQLL3vSnfFl5bj4sL75qC9D6EdjemIdbtkuUmIheWpuiiylSY01JxJE7+zGrOWDZrp1WxOuDntvKrHQ==", - "dev": true, - "requires": { - "@typescript-eslint/types": "7.12.0", - "eslint-visitor-keys": "^3.4.3" - } - }, - "@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", - "dev": true - }, - "abbrev": { - "version": "1.1.1", - "dev": true - }, - "abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "requires": { - "event-target-shim": "^5.0.0" - } - }, - "accepts": { - "version": "1.3.8", - "requires": { - "mime-types": "~2.1.34", - "negotiator": "0.6.3" - } - }, - "acorn": { - "version": "8.11.3", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.11.3.tgz", - "integrity": "sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg==", - "dev": true - }, - "acorn-jsx": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.2.tgz", - "integrity": "sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==", - "dev": true, - "requires": {} - }, - "acorn-walk": { - "version": "8.2.0", - "dev": true - }, - "agent-base": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", - "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "requires": { - "debug": "^4.3.4" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, - "aggregate-error": { - "version": "3.1.0", - "dev": true, - "requires": { - "clean-stack": "^2.0.0", - "indent-string": "^4.0.0" - } - }, - "ajv": { - "version": "8.12.0", - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "ajv-formats": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", - "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", - "requires": { - "ajv": "^8.0.0" - } - }, - "ansi-colors": { - "version": "4.1.1", - "dev": true - }, - "ansi-regex": { - "version": "5.0.1", - "devOptional": true - }, - "ansi-styles": { - "version": "4.3.0", - "dev": true, - "requires": { - "color-convert": "^2.0.1" - } - }, - "anymatch": { - "version": "3.1.3", - "dev": true, - "requires": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - } - }, - "append-transform": { - "version": "2.0.0", - "dev": true, - "requires": { - "default-require-extensions": "^3.0.0" - } - }, - "aproba": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", - "optional": true - }, - "archy": { - "version": "1.0.0", - "dev": true - }, - "are-we-there-yet": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.7.tgz", - "integrity": "sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==", - "optional": true, - "requires": { - "delegates": "^1.0.0", - "readable-stream": "^2.0.6" - } - }, - "arg": { - "version": "4.1.3", - "dev": true - }, - "argparse": { - "version": "2.0.1", - "dev": true - }, - "array-flatten": { - "version": "1.1.1" - }, - "array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true - }, - "arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" - }, - "assertion-error": { - "version": "1.1.0", - "dev": true - }, - "async": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/async/-/async-2.6.4.tgz", - "integrity": "sha512-mzo5dfJYwAn29PeiJ0zvwTo04zj8HDJj0Mn8TD7sno7q12prdbnasKJHhkm2c1LgrhlJ0teaea8860oxi51mGA==", - "requires": { - "lodash": "^4.17.14" - } - }, - "async-retry": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/async-retry/-/async-retry-1.3.3.tgz", - "integrity": "sha512-wfr/jstw9xNi/0teMHrRW7dsz3Lt5ARhYNZ2ewpadnhaIp5mbALhOAP+EAdsC7t4Z6wqsDVv9+W6gm1Dk9mEyw==", - "requires": { - "retry": "0.13.1" - } - }, - "asynckit": { - "version": "0.4.0" - }, - "available-typed-arrays": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", - "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" - }, - "aws-sdk": { - "version": "2.1472.0", - "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1472.0.tgz", - "integrity": "sha512-U7kAHRbvTy753IXKV8Oom/AqlqnsbXG+Kw5gRbKi6VcsZ3hR/EpNMzdRXTWO5U415bnLWGo8WAqIz67PIaaKsw==", - "requires": { - "buffer": "4.9.2", - "events": "1.1.1", - "ieee754": "1.1.13", - "jmespath": "0.16.0", - "querystring": "0.2.0", - "sax": "1.2.1", - "url": "0.10.3", - "util": "^0.12.4", - "uuid": "8.0.0", - "xml2js": "0.5.0" - }, - "dependencies": { - "buffer": { - "version": "4.9.2", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", - "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4", - "isarray": "^1.0.0" - } - }, - "events": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", - "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==" - }, - "uuid": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", - "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==" - } - } - }, - "axios": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz", - "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==", - "requires": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, - "balanced-match": { - "version": "1.0.2" - }, - "base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" - }, - "big-integer": { - "version": "1.6.52", - "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.52.tgz", - "integrity": "sha512-QxD8cf2eVqJOOz63z6JIN9BzvVs/dlySa5HGSBH5xtR8dPteIRQnBxxKqkNTiT6jbDTF6jAfrd4oMcND9RGbQg==" - }, - "bignumber.js": { - "version": "9.1.2", - "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.2.tgz", - "integrity": "sha512-2/mKyZH9K85bzOEfhXDBFZTGd1CTs+5IHpeFQo9luiBG7hghdC851Pj2WAhb6E3R6b9tZj/XKhbg4fum+Kepug==" - }, - "binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha512-D4H1y5KYwpJgK8wk1Cue5LLPgmwHKYSChkbspQg5JtVuR5ulGckxfR62H3AE9UDkdMC8yyXlqYihuz3Aqg2XZg==", - "requires": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - } - }, - "binary-extensions": { - "version": "2.2.0", - "dev": true - }, - "bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "optional": true, - "requires": { - "file-uri-to-path": "1.0.0" - } - }, - "bintrees": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.2.tgz", - "integrity": "sha512-VOMgTMwjAaUG580SXn3LacVgjurrbMme7ZZNYGSSV7mmtY6QQRh0Eg3pwIcntQ77DErK1L0NxkbetjcoXzVwKw==" - }, - "bl": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/bl/-/bl-2.2.1.tgz", - "integrity": "sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g==", - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - }, - "body-parser": { - "version": "1.20.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", - "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", - "requires": { - "bytes": "3.1.2", - "content-type": "~1.0.5", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.2", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - } - }, - "bowser": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", - "integrity": "sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==" - }, - "brace-expansion": { - "version": "1.1.11", - "requires": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "requires": { - "fill-range": "^7.1.1" - } - }, - "browser-stdout": { - "version": "1.3.1", - "dev": true - }, - "browserslist": { - "version": "4.21.5", - "dev": true, - "requires": { - "caniuse-lite": "^1.0.30001449", - "electron-to-chromium": "^1.4.284", - "node-releases": "^2.0.8", - "update-browserslist-db": "^1.0.10" - } - }, - "buffer": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", - "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, - "buffer-alloc": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/buffer-alloc/-/buffer-alloc-1.2.0.tgz", - "integrity": "sha512-CFsHQgjtW1UChdXgbyJGtnm+O/uLQeZdtbDo8mfUgYXCHSM1wgrVxXm6bSyrUuErEb+4sYVGCzASBRot7zyrow==", - "optional": true, - "requires": { - "buffer-alloc-unsafe": "^1.1.0", - "buffer-fill": "^1.0.0" - } - }, - "buffer-alloc-unsafe": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/buffer-alloc-unsafe/-/buffer-alloc-unsafe-1.1.0.tgz", - "integrity": "sha512-TEM2iMIEQdJ2yjPJoSIsldnleVaAk1oW3DBVUykyOLsEsFmEc9kn+SFFPz+gl54KQNxlDnAwCXosOS9Okx2xAg==", - "optional": true - }, - "buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==" - }, - "buffer-equal-constant-time": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", - "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" - }, - "buffer-fill": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", - "integrity": "sha512-T7zexNBwiiaCOGDg9xNX9PBmjrubblRkENuptryuI64URkXDFum9il/JGL8Lm8wYfAXpredVXXZz7eMHilimiQ==", - "optional": true - }, - "buffermaker": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/buffermaker/-/buffermaker-1.2.1.tgz", - "integrity": "sha512-IdnyU2jDHU65U63JuVQNTHiWjPRH0CS3aYd/WPaEwyX84rFdukhOduAVb1jwUScmb5X0JWPw8NZOrhoLMiyAHQ==", - "requires": { - "long": "1.1.2" - } - }, - "buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ==" - }, - "bytes": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", - "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" - }, - "caching-transform": { - "version": "4.0.0", - "dev": true, - "requires": { - "hasha": "^5.0.0", - "make-dir": "^3.0.0", - "package-hash": "^4.0.0", - "write-file-atomic": "^3.0.0" - } - }, - "call-bind": { - "version": "1.0.2", - "requires": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" - } - }, - "callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true - }, - "camelcase": { - "version": "6.3.0", - "dev": true - }, - "caniuse-lite": { - "version": "1.0.30001466", - "dev": true - }, - "chai": { - "version": "4.3.7", - "dev": true, - "requires": { - "assertion-error": "^1.1.0", - "check-error": "^1.0.2", - "deep-eql": "^4.1.2", - "get-func-name": "^2.0.0", - "loupe": "^2.3.1", - "pathval": "^1.1.1", - "type-detect": "^4.0.5" - } - }, - "chai-as-promised": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/chai-as-promised/-/chai-as-promised-7.1.1.tgz", - "integrity": "sha512-azL6xMoi+uxu6z4rhWQ1jbdUhOMhis2PvscD/xjLqNMkv3BPPp2JyyuTHOrf9BOosGpNQ11v6BKv/g57RXbiaA==", - "dev": true, - "requires": { - "check-error": "^1.0.2" - } - }, - "chai-http": { - "version": "4.3.0", - "dev": true, - "requires": { - "@types/chai": "4", - "@types/superagent": "^3.8.3", - "cookiejar": "^2.1.1", - "is-ip": "^2.0.0", - "methods": "^1.1.2", - "qs": "^6.5.1", - "superagent": "^3.7.0" - } - }, - "chai-spies": { - "version": "1.0.0", - "dev": true, - "requires": {} - }, - "chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha512-75kWfWt6MEKNC8xYXIdRpDehRYY/tNSgwKaJq+dbbDcxORuVrrQ+SEHoWsniVn9XPYfP4gmdWIeDk/4YNp1rNQ==", - "requires": { - "traverse": ">=0.3.0 <0.4" - } - }, - "chalk": { - "version": "4.1.2", - "dev": true, - "requires": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "dependencies": { - "supports-color": { - "version": "7.2.0", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, - "check-error": { - "version": "1.0.2", - "dev": true - }, - "chokidar": { - "version": "3.5.3", - "dev": true, - "requires": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "fsevents": "~2.3.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - } - }, - "chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", - "optional": true - }, - "clean-stack": { - "version": "2.2.0", - "dev": true - }, - "cliui": { - "version": "7.0.4", - "dev": true, - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^7.0.0" - } - }, - "code-point-at": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz", - "integrity": "sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==", - "optional": true - }, - "color": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", - "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", - "requires": { - "color-convert": "^1.9.3", - "color-string": "^1.6.0" - }, - "dependencies": { - "color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "requires": { - "color-name": "1.1.3" - } - }, - "color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" - } - } - }, - "color-convert": { - "version": "2.0.1", - "dev": true, - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4" - }, - "color-string": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", - "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", - "requires": { - "color-name": "^1.0.0", - "simple-swizzle": "^0.2.2" - } - }, - "colorette": { - "version": "2.0.19" - }, - "colors": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", - "integrity": "sha512-pFGrxThWcWQ2MsAz6RtgeWe4NK2kUE1WfsrvvlctdII745EW9I0yflqhe7++M5LEc7bV2c/9/5zc8sFcpL0Drw==" - }, - "colorspace": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", - "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", - "requires": { - "color": "^3.1.3", - "text-hex": "1.0.x" - } - }, - "combined-stream": { - "version": "1.0.8", - "requires": { - "delayed-stream": "~1.0.0" - } - }, - "commander": { - "version": "9.5.0" - }, - "commondir": { - "version": "1.0.1", - "dev": true - }, - "component-emitter": { - "version": "1.3.0", - "dev": true - }, - "compressible": { - "version": "2.0.18", - "resolved": "https://registry.npmjs.org/compressible/-/compressible-2.0.18.tgz", - "integrity": "sha512-AF3r7P5dWxL8MxyITRMlORQNaOA2IkAFaTr4k7BUumjPtRpGDTZpl0Pb1XCO6JeDCBdp126Cgs9sMxqSjgYyRg==", - "requires": { - "mime-db": ">= 1.43.0 < 2" - } - }, - "compression": { - "version": "1.7.4", - "resolved": "https://registry.npmjs.org/compression/-/compression-1.7.4.tgz", - "integrity": "sha512-jaSIDzP9pZVS4ZfQ+TzvtiWhdpFhE2RDHz8QJkpX9SIpLq88VueF5jJw6t+6CUQcAoA6t+x89MLrWAqpfDE8iQ==", - "requires": { - "accepts": "~1.3.5", - "bytes": "3.0.0", - "compressible": "~2.0.16", - "debug": "2.6.9", - "on-headers": "~1.0.2", - "safe-buffer": "5.1.2", - "vary": "~1.1.2" - }, - "dependencies": { - "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha512-pMhOfFDPiv9t5jjIXkHosWmkSyQbvsgEVNkz0ERHbuLh2T/7j4Mqqpz523Fe8MVY89KC6Sh/QfS2sM+SjgFDcw==" - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - } - } - }, - "concat-map": { - "version": "0.0.1" - }, - "console-control-strings": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==", - "optional": true - }, - "content-disposition": { - "version": "0.5.4", - "requires": { - "safe-buffer": "5.2.1" - } - }, - "content-type": { - "version": "1.0.5" - }, - "convert-source-map": { - "version": "1.9.0", - "dev": true - }, - "cookie": { - "version": "0.5.0" - }, - "cookie-signature": { - "version": "1.0.6" - }, - "cookiejar": { - "version": "2.1.4", - "dev": true - }, - "core-util-is": { - "version": "1.0.3" - }, - "create-require": { - "version": "1.1.1", - "dev": true - }, - "cross-spawn": { - "version": "7.0.3", - "dev": true, - "requires": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - } - }, - "cycle": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", - "integrity": "sha512-TVF6svNzeQCOpjCqsy0/CSy8VgObG3wXusJ73xW2GbG5rGx7lC8zxDSURicsXI2UsGdi2L0QNRCi745/wUDvsA==" - }, - "date-format": { - "version": "4.0.14", - "resolved": "https://registry.npmjs.org/date-format/-/date-format-4.0.14.tgz", - "integrity": "sha512-39BOQLs9ZjKh0/patS9nrT8wc3ioX3/eA/zgbKNopnF2wCqJEoxywwwElATYvRsXdnOxA/OQeQoFZ3rFjVajhg==" - }, - "dateformat": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-2.0.0.tgz", - "integrity": "sha512-k6+FtJ8RoNx9V0yHo6lURQWlFqc8wbo0t/kocHvfMJQiXTW+izR6bubmB9HeWuUiZFtpMrAm+wPMGmGhfPbNlQ==" - }, - "debug": { - "version": "2.6.9", - "requires": { - "ms": "2.0.0" - } - }, - "decamelize": { - "version": "4.0.0", - "dev": true - }, - "decompress-response": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-3.3.0.tgz", - "integrity": "sha512-BzRPQuY1ip+qDonAOz42gRm/pg9F768C+npV/4JOsxRC2sq+Rlk+Q4ZCAsOhnIaMrgarILY+RMUIvMmmX1qAEA==", - "optional": true, - "requires": { - "mimic-response": "^1.0.0" - } - }, - "deep-eql": { - "version": "4.1.3", - "dev": true, - "requires": { - "type-detect": "^4.0.0" - } - }, - "deep-extend": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", - "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", - "optional": true - }, - "deep-is": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true - }, - "default-require-extensions": { - "version": "3.0.1", - "dev": true, - "requires": { - "strip-bom": "^4.0.0" - } - }, - "delayed-stream": { - "version": "1.0.0" - }, - "delegates": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz", - "integrity": "sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==", - "optional": true - }, - "denque": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/denque/-/denque-1.5.1.tgz", - "integrity": "sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw==" - }, - "depd": { - "version": "2.0.0" - }, - "destroy": { - "version": "1.2.0" - }, - "detect-libc": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", - "integrity": "sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==", - "optional": true - }, - "diff": { - "version": "5.0.0", - "dev": true - }, - "diff-json": { - "version": "2.0.0", - "dev": true, - "requires": { - "lodash.difference": ">= 4.0.0", - "lodash.find": ">= 4.0.0", - "lodash.intersection": ">= 4.0.0", - "lodash.keyby": ">= 4.0.0" - } - }, - "dir-glob": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz", - "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==", - "dev": true, - "requires": { - "path-type": "^4.0.0" - } - }, - "doctrine": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", - "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", - "dev": true, - "requires": { - "esutils": "^2.0.2" - } - }, - "dottie": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/dottie/-/dottie-2.0.6.tgz", - "integrity": "sha512-iGCHkfUc5kFekGiqhe8B/mdaurD+lakO9txNnTvKtA6PISrw86LgqHvRzWYPyoE2Ph5aMIrCw9/uko6XHTKCwA==" - }, - "duplexify": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-4.1.3.tgz", - "integrity": "sha512-M3BmBhwJRZsSx38lZyhE53Csddgzl5R7xGJNk7CVddZD6CcmwMCH8J+7AprIrQKH7TonKxaCjcv27Qmf+sQ+oA==", - "requires": { - "end-of-stream": "^1.4.1", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1", - "stream-shift": "^1.0.2" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, - "ecdsa-sig-formatter": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", - "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "ee-first": { - "version": "1.1.1" - }, - "electron-to-chromium": { - "version": "1.4.330", - "dev": true - }, - "emoji-regex": { - "version": "8.0.0", - "devOptional": true - }, - "enabled": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", - "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==" - }, - "encodeurl": { - "version": "1.0.2" - }, - "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "requires": { - "once": "^1.4.0" - } - }, - "es6-error": { - "version": "4.1.1", - "dev": true - }, - "escalade": { - "version": "3.1.1" - }, - "escape-html": { - "version": "1.0.3" - }, - "escape-string-regexp": { - "version": "4.0.0", - "dev": true - }, - "eslint": { - "version": "8.57.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.57.0.tgz", - "integrity": "sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ==", - "dev": true, - "requires": { - "@eslint-community/eslint-utils": "^4.2.0", - "@eslint-community/regexpp": "^4.6.1", - "@eslint/eslintrc": "^2.1.4", - "@eslint/js": "8.57.0", - "@humanwhocodes/config-array": "^0.11.14", - "@humanwhocodes/module-importer": "^1.0.1", - "@nodelib/fs.walk": "^1.2.8", - "@ungap/structured-clone": "^1.2.0", - "ajv": "^6.12.4", - "chalk": "^4.0.0", - "cross-spawn": "^7.0.2", - "debug": "^4.3.2", - "doctrine": "^3.0.0", - "escape-string-regexp": "^4.0.0", - "eslint-scope": "^7.2.2", - "eslint-visitor-keys": "^3.4.3", - "espree": "^9.6.1", - "esquery": "^1.4.2", - "esutils": "^2.0.2", - "fast-deep-equal": "^3.1.3", - "file-entry-cache": "^6.0.1", - "find-up": "^5.0.0", - "glob-parent": "^6.0.2", - "globals": "^13.19.0", - "graphemer": "^1.4.0", - "ignore": "^5.2.0", - "imurmurhash": "^0.1.4", - "is-glob": "^4.0.0", - "is-path-inside": "^3.0.3", - "js-yaml": "^4.1.0", - "json-stable-stringify-without-jsonify": "^1.0.1", - "levn": "^0.4.1", - "lodash.merge": "^4.6.2", - "minimatch": "^3.1.2", - "natural-compare": "^1.4.0", - "optionator": "^0.9.3", - "strip-ansi": "^6.0.1", - "text-table": "^0.2.0" - }, - "dependencies": { - "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", - "uri-js": "^4.2.2" - } - }, - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "glob-parent": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", - "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", - "dev": true, - "requires": { - "is-glob": "^4.0.3" - } - }, - "globals": { - "version": "13.24.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz", - "integrity": "sha512-AhO5QUcj8llrbG09iWhPU2B204J1xnPeL8kQmVorSsy+Sjj1sk8gIyh6cUocGmH4L0UuhAJy+hJMRA4mgA4mFQ==", - "dev": true, - "requires": { - "type-fest": "^0.20.2" - } - }, - "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", - "dev": true - }, - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, - "type-fest": { - "version": "0.20.2", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.20.2.tgz", - "integrity": "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==", - "dev": true - } - } - }, - "eslint-scope": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.2.2.tgz", - "integrity": "sha512-dOt21O7lTMhDM+X9mB4GX+DZrZtCUJPL/wlcTqxyrx5IvO0IYtILdtrQGQp+8n5S0gwSVmOf9NQrjMOgfQZlIg==", - "dev": true, - "requires": { - "esrecurse": "^4.3.0", - "estraverse": "^5.2.0" - } - }, - "eslint-visitor-keys": { - "version": "3.4.3", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", - "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", - "dev": true - }, - "esm": { - "version": "3.2.25" - }, - "espree": { - "version": "9.6.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", - "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", - "dev": true, - "requires": { - "acorn": "^8.9.0", - "acorn-jsx": "^5.3.2", - "eslint-visitor-keys": "^3.4.1" - } - }, - "esquery": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.5.0.tgz", - "integrity": "sha512-YQLXUplAwJgCydQ78IMJywZCceoqk1oH01OERdSAJc/7U2AylwjhSCLDEtqwg811idIS/9fIU5GjG73IgjKMVg==", - "dev": true, - "requires": { - "estraverse": "^5.1.0" - } - }, - "esrecurse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", - "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", - "dev": true, - "requires": { - "estraverse": "^5.2.0" - } - }, - "estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true - }, - "esutils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true - }, - "etag": { - "version": "1.8.1" - }, - "event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" - }, - "events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" - }, - "expand-template": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", - "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", - "optional": true - }, - "express": { - "version": "4.18.2", - "requires": { - "accepts": "~1.3.8", - "array-flatten": "1.1.1", - "body-parser": "1.20.1", - "content-disposition": "0.5.4", - "content-type": "~1.0.4", - "cookie": "0.5.0", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "2.0.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "finalhandler": "1.2.0", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "merge-descriptors": "1.0.1", - "methods": "~1.1.2", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "path-to-regexp": "0.1.7", - "proxy-addr": "~2.0.7", - "qs": "6.11.0", - "range-parser": "~1.2.1", - "safe-buffer": "5.2.1", - "send": "0.18.0", - "serve-static": "1.15.0", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "type-is": "~1.6.18", - "utils-merge": "1.0.1", - "vary": "~1.1.2" - }, - "dependencies": { - "body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", - "requires": { - "bytes": "3.1.2", - "content-type": "~1.0.4", - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "on-finished": "2.4.1", - "qs": "6.11.0", - "raw-body": "2.5.1", - "type-is": "~1.6.18", - "unpipe": "1.0.0" - } - }, - "raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", - "requires": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - } - } - } - }, - "extend": { - "version": "3.0.2" - }, - "eyes": { - "version": "0.1.8", - "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", - "integrity": "sha512-GipyPsXO1anza0AOZdy69Im7hGFCNB7Y/NGjDlZGJ3GJJLtwNSb2vrzYrTYJRrRloVx7pl+bhUaTB8yiccPvFQ==" - }, - "fast-deep-equal": { - "version": "3.1.3" - }, - "fast-glob": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.2.tgz", - "integrity": "sha512-oX2ruAFQwf/Orj8m737Y5adxDQO0LAB7/S5MnxCdTNDd4p6BsyIVsv9JQsATbTSq8KHRpLwIHbVlUNatxd+1Ow==", - "dev": true, - "requires": { - "@nodelib/fs.stat": "^2.0.2", - "@nodelib/fs.walk": "^1.2.3", - "glob-parent": "^5.1.2", - "merge2": "^1.3.0", - "micromatch": "^4.0.4" - } - }, - "fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true - }, - "fast-levenshtein": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true - }, - "fast-safe-stringify": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/fast-safe-stringify/-/fast-safe-stringify-2.1.1.tgz", - "integrity": "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==" - }, - "fast-xml-parser": { - "version": "4.2.5", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz", - "integrity": "sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g==", - "requires": { - "strnum": "^1.0.5" - } - }, - "fastq": { - "version": "1.17.1", - "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.17.1.tgz", - "integrity": "sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w==", - "dev": true, - "requires": { - "reusify": "^1.0.4" - } - }, - "fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g==", - "requires": { - "pend": "~1.2.0" - } - }, - "fecha": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz", - "integrity": "sha512-lUGBnIamTAwk4znq5BcqsDaxSmZ9nDVJaij6NvRt/Tg4R69gERA+otPKbS86ROw9nxVMw2/mp1fnaiWqbs6Sdg==" - }, - "file-entry-cache": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-6.0.1.tgz", - "integrity": "sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==", - "dev": true, - "requires": { - "flat-cache": "^3.0.4" - } - }, - "file-stream-rotator": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/file-stream-rotator/-/file-stream-rotator-0.2.1.tgz", - "integrity": "sha512-Qf6xva6g0fWqI3LR48eOa8ubF+AP6ftLUSt9Uin6XadP5s3d4vD0J52l/JjmMPiFleCtsVJJ6cyMU2GNRLXQIQ==", - "requires": { - "moment": "^2.11.2" - } - }, - "file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "optional": true - }, - "fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "requires": { - "to-regex-range": "^5.0.1" - } - }, - "finalhandler": { - "version": "1.2.0", - "requires": { - "debug": "2.6.9", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "on-finished": "2.4.1", - "parseurl": "~1.3.3", - "statuses": "2.0.1", - "unpipe": "~1.0.0" - } - }, - "find-cache-dir": { - "version": "3.3.2", - "dev": true, - "requires": { - "commondir": "^1.0.1", - "make-dir": "^3.0.2", - "pkg-dir": "^4.1.0" - } - }, - "find-up": { - "version": "5.0.0", - "dev": true, - "requires": { - "locate-path": "^6.0.0", - "path-exists": "^4.0.0" - } - }, - "flat": { - "version": "5.0.2", - "dev": true - }, - "flat-cache": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-3.2.0.tgz", - "integrity": "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw==", - "dev": true, - "requires": { - "flatted": "^3.2.9", - "keyv": "^4.5.3", - "rimraf": "^3.0.2" - } - }, - "flatted": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.1.tgz", - "integrity": "sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw==" - }, - "fn.name": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", - "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" - }, - "follow-redirects": { - "version": "1.15.2" - }, - "for-each": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", - "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "requires": { - "is-callable": "^1.1.3" - } - }, - "foreground-child": { - "version": "2.0.0", - "dev": true, - "requires": { - "cross-spawn": "^7.0.0", - "signal-exit": "^3.0.2" - } - }, - "form-data": { - "version": "4.0.0", - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "mime-types": "^2.1.12" - } - }, - "formidable": { - "version": "1.2.6", - "dev": true - }, - "forwarded": { - "version": "0.2.0" - }, - "fresh": { - "version": "0.5.2" - }, - "fromentries": { - "version": "1.3.2", - "dev": true - }, - "fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", - "optional": true - }, - "fs-extra": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", - "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", - "requires": { - "graceful-fs": "^4.2.0", - "jsonfile": "^4.0.0", - "universalify": "^0.1.0" - } - }, - "fs.realpath": { - "version": "1.0.0", - "dev": true - }, - "fsevents": { - "version": "2.3.2", - "dev": true, - "optional": true - }, - "function-bind": { - "version": "1.1.1" - }, - "gauge": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", - "integrity": "sha512-14x4kjc6lkD3ltw589k0NrPD6cCNTD6CWoVUNpB85+DrtONoZn+Rug6xZU5RvSC4+TZPxA5AnBibQYAvZn41Hg==", - "optional": true, - "requires": { - "aproba": "^1.0.3", - "console-control-strings": "^1.0.0", - "has-unicode": "^2.0.0", - "object-assign": "^4.1.0", - "signal-exit": "^3.0.0", - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wide-align": "^1.1.0" - }, - "dependencies": { - "ansi-regex": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "optional": true - }, - "is-fullwidth-code-point": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", - "integrity": "sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==", - "optional": true, - "requires": { - "number-is-nan": "^1.0.0" - } - }, - "string-width": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", - "integrity": "sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==", - "optional": true, - "requires": { - "code-point-at": "^1.0.0", - "is-fullwidth-code-point": "^1.0.0", - "strip-ansi": "^3.0.0" - } - }, - "strip-ansi": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", - "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", - "optional": true, - "requires": { - "ansi-regex": "^2.0.0" - } - } - } - }, - "gaxios": { - "version": "6.6.0", - "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-6.6.0.tgz", - "integrity": "sha512-bpOZVQV5gthH/jVCSuYuokRo2bTKOcuBiVWpjmTn6C5Agl5zclGfTljuGsQZxwwDBkli+YhZhP4TdlqTnhOezQ==", - "requires": { - "extend": "^3.0.2", - "https-proxy-agent": "^7.0.1", - "is-stream": "^2.0.0", - "node-fetch": "^2.6.9", - "uuid": "^9.0.1" - }, - "dependencies": { - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "gcp-metadata": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-6.1.0.tgz", - "integrity": "sha512-Jh/AIwwgaxan+7ZUUmRLCjtchyDiqh4KjBJ5tW3plBZb5iL/BPcso8A5DlzeD9qlw0duCamnNdpFjxwaT0KyKg==", - "requires": { - "gaxios": "^6.0.0", - "json-bigint": "^1.0.0" - } - }, - "gensync": { - "version": "1.0.0-beta.2", - "dev": true - }, - "get-caller-file": { - "version": "2.0.5", - "dev": true - }, - "get-func-name": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.2.tgz", - "integrity": "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ==", - "dev": true - }, - "get-intrinsic": { - "version": "1.2.0", - "requires": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" - } - }, - "get-package-type": { - "version": "0.1.0" - }, - "getopts": { - "version": "2.3.0" - }, - "github-from-package": { - "version": "0.0.0", - "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", - "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", - "optional": true - }, - "glob": { - "version": "7.2.0", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "dependencies": { - "minimatch": { - "version": "3.1.2", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - } - } - }, - "glob-parent": { - "version": "5.1.2", - "dev": true, - "requires": { - "is-glob": "^4.0.1" - } - }, - "globals": { - "version": "11.12.0", - "dev": true - }, - "globby": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/globby/-/globby-11.1.0.tgz", - "integrity": "sha512-jhIXaOzy1sb8IyocaruWSn1TjmnBVs8Ayhcy83rmxNJ8q2uWKCAj3CnJY+KpGSXCueAPc0i05kVvVKtP1t9S3g==", - "dev": true, - "requires": { - "array-union": "^2.1.0", - "dir-glob": "^3.0.1", - "fast-glob": "^3.2.9", - "ignore": "^5.2.0", - "merge2": "^1.4.1", - "slash": "^3.0.0" - } - }, - "google-auth-library": { - "version": "9.10.0", - "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.10.0.tgz", - "integrity": "sha512-ol+oSa5NbcGdDqA+gZ3G3mev59OHBZksBTxY/tYwjtcp1H/scAFwJfSQU9/1RALoyZ7FslNbke8j4i3ipwlyuQ==", - "requires": { - "base64-js": "^1.3.0", - "ecdsa-sig-formatter": "^1.0.11", - "gaxios": "^6.1.1", - "gcp-metadata": "^6.1.0", - "gtoken": "^7.0.0", - "jws": "^4.0.0" - } - }, - "gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "requires": { - "get-intrinsic": "^1.1.3" - } - }, - "graceful-fs": { - "version": "4.2.10" - }, - "graphemer": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/graphemer/-/graphemer-1.4.0.tgz", - "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", - "dev": true - }, - "gtoken": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-7.1.0.tgz", - "integrity": "sha512-pCcEwRi+TKpMlxAQObHDQ56KawURgyAf6jtIY046fJ5tIv3zDe/LEIubckAO8fj6JnAxLdmWkUfNyulQ2iKdEw==", - "requires": { - "gaxios": "^6.0.0", - "jws": "^4.0.0" - } - }, - "has": { - "version": "1.0.3", - "requires": { - "function-bind": "^1.1.1" - } - }, - "has-flag": { - "version": "4.0.0", - "dev": true - }, - "has-symbols": { - "version": "1.0.3" - }, - "has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", - "requires": { - "has-symbols": "^1.0.2" - } - }, - "has-unicode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", - "optional": true - }, - "hasha": { - "version": "5.2.2", - "dev": true, - "requires": { - "is-stream": "^2.0.0", - "type-fest": "^0.8.0" - } - }, - "he": { - "version": "1.2.0", - "dev": true - }, - "html-entities": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/html-entities/-/html-entities-2.5.2.tgz", - "integrity": "sha512-K//PSRMQk4FZ78Kyau+mZurHn3FH0Vwr+H36eE0rPbeYkRRi9YxceYPhuN60UwWorxyKHhqoAJl2OFKa4BVtaA==" - }, - "html-escaper": { - "version": "2.0.2", - "dev": true - }, - "http-errors": { - "version": "2.0.0", - "requires": { - "depd": "2.0.0", - "inherits": "2.0.4", - "setprototypeof": "1.2.0", - "statuses": "2.0.1", - "toidentifier": "1.0.1" - } - }, - "http-proxy-agent": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", - "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "requires": { - "agent-base": "^7.1.0", - "debug": "^4.3.4" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, - "http-status": { - "version": "1.6.2" - }, - "https-proxy-agent": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz", - "integrity": "sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg==", - "requires": { - "agent-base": "^7.0.2", - "debug": "4" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, - "iconv-lite": { - "version": "0.4.24", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", - "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "requires": { - "safer-buffer": ">= 2.1.2 < 3" - } - }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, - "ignore": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.1.tgz", - "integrity": "sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw==", - "dev": true - }, - "ignore-by-default": { - "version": "1.0.1", - "dev": true - }, - "import-fresh": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.0.tgz", - "integrity": "sha512-veYYhQa+D1QBKznvhUHxb8faxlrwUnxseDAbAp457E0wLNio2bOSKnjYDhMj+YiAq61xrMGhQk9iXVk5FzgQMw==", - "dev": true, - "requires": { - "parent-module": "^1.0.0", - "resolve-from": "^4.0.0" - }, - "dependencies": { - "resolve-from": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", - "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", - "dev": true - } - } - }, - "imurmurhash": { - "version": "0.1.4", - "dev": true - }, - "indent-string": { - "version": "4.0.0", - "dev": true - }, - "inflection": { - "version": "1.13.4", - "resolved": "https://registry.npmjs.org/inflection/-/inflection-1.13.4.tgz", - "integrity": "sha512-6I/HUDeYFfuNCVS3td055BaXBwKYuzw7K3ExVMStBowKo9oOAMJIXIHvdyR3iboTCp1b+1i5DSkIZTcwIktuDw==" - }, - "inflight": { - "version": "1.0.6", - "dev": true, - "requires": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "inherits": { - "version": "2.0.4" - }, - "ini": { - "version": "1.3.8", - "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", - "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", - "optional": true - }, - "interpret": { - "version": "2.2.0" - }, - "ip-regex": { - "version": "2.1.0", - "dev": true - }, - "ipaddr.js": { - "version": "1.9.1" - }, - "is-arguments": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", - "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", - "requires": { - "call-bind": "^1.0.2", - "has-tostringtag": "^1.0.0" - } - }, - "is-arrayish": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", - "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==" - }, - "is-binary-path": { - "version": "2.1.0", - "dev": true, - "requires": { - "binary-extensions": "^2.0.0" - } - }, - "is-callable": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", - "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==" - }, - "is-core-module": { - "version": "2.11.0", - "requires": { - "has": "^1.0.3" - } - }, - "is-extglob": { - "version": "2.1.1", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "3.0.0", - "devOptional": true - }, - "is-generator-function": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", - "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", - "requires": { - "has-tostringtag": "^1.0.0" - } - }, - "is-glob": { - "version": "4.0.3", - "dev": true, - "requires": { - "is-extglob": "^2.1.1" - } - }, - "is-ip": { - "version": "2.0.0", - "dev": true, - "requires": { - "ip-regex": "^2.0.0" - } - }, - "is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true - }, - "is-path-inside": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/is-path-inside/-/is-path-inside-3.0.3.tgz", - "integrity": "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ==", - "dev": true - }, - "is-plain-obj": { - "version": "2.1.0", - "dev": true - }, - "is-stream": { - "version": "2.0.1" - }, - "is-typed-array": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.12.tgz", - "integrity": "sha512-Z14TF2JNG8Lss5/HMqt0//T9JeHXttXy5pH/DBU4vi98ozO2btxzq9MwYDZYnKwU8nRsz/+GVFVRDq3DkVuSPg==", - "requires": { - "which-typed-array": "^1.1.11" - } - }, - "is-typedarray": { - "version": "1.0.0", - "dev": true - }, - "is-unicode-supported": { - "version": "0.1.0", - "dev": true - }, - "is-windows": { - "version": "1.0.2", - "dev": true - }, - "isarray": { - "version": "1.0.0" - }, - "isexe": { - "version": "2.0.0", - "dev": true - }, - "isstream": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", - "integrity": "sha512-Yljz7ffyPbrLpLngrMtZ7NduUgVvi6wG9RJ9IUcyCd59YQ911PBJphODUcbOVbqYfxe1wuYf/LJ8PauMRwsM/g==" - }, - "istanbul-lib-coverage": { - "version": "3.2.0", - "dev": true - }, - "istanbul-lib-hook": { - "version": "3.0.0", - "dev": true, - "requires": { - "append-transform": "^2.0.0" - } - }, - "istanbul-lib-instrument": { - "version": "4.0.3", - "dev": true, - "requires": { - "@babel/core": "^7.7.5", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.0.0", - "semver": "^7.5.3" - } - }, - "istanbul-lib-processinfo": { - "version": "2.0.3", - "dev": true, - "requires": { - "archy": "^1.0.0", - "cross-spawn": "^7.0.3", - "istanbul-lib-coverage": "^3.2.0", - "p-map": "^3.0.0", - "rimraf": "^3.0.0", - "uuid": "^8.3.2" - }, - "dependencies": { - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true - } - } - }, - "istanbul-lib-report": { - "version": "3.0.0", - "dev": true, - "requires": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^3.0.0", - "supports-color": "^7.1.0" - }, - "dependencies": { - "supports-color": { - "version": "7.2.0", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, - "istanbul-lib-source-maps": { - "version": "4.0.1", - "dev": true, - "requires": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "istanbul-reports": { - "version": "3.1.5", - "dev": true, - "requires": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - } - }, - "jmespath": { - "version": "0.16.0", - "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", - "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" - }, - "js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true - }, - "js-yaml": { - "version": "4.1.0", - "dev": true, - "requires": { - "argparse": "^2.0.1" - } - }, - "jsesc": { - "version": "2.5.2", - "dev": true - }, - "json-bigint": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", - "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", - "requires": { - "bignumber.js": "^9.0.0" - } - }, - "json-buffer": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", - "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", - "dev": true - }, - "json-schema-traverse": { - "version": "1.0.0" - }, - "json-stable-stringify-without-jsonify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", - "integrity": "sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==", - "dev": true - }, - "json-stringify-safe": { - "version": "5.0.1" - }, - "json5": { - "version": "2.2.3", - "dev": true - }, - "jsonfile": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", - "integrity": "sha512-m6F1R3z8jjlf2imQHS2Qez5sjKWQzbuuhuJ/FKYFRZvPE3PuHcSMVZzfsLhGVOkfd20obL5SWEBew5ShlquNxg==", - "requires": { - "graceful-fs": "^4.1.6" - } - }, - "just-extend": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", - "integrity": "sha512-cYofQu2Xpom82S6qD778jBDpwvvy39s1l/hrYij2u9AMdQcGRpaBu6kY4mVhuno5kJVi1DAz4aiphA2WI1/OAw==", - "dev": true - }, - "jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", - "requires": { - "buffer-equal-constant-time": "1.0.1", - "ecdsa-sig-formatter": "1.0.11", - "safe-buffer": "^5.0.1" - } - }, - "jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", - "requires": { - "jwa": "^2.0.0", - "safe-buffer": "^5.0.1" - } - }, - "kafka-node": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/kafka-node/-/kafka-node-5.0.0.tgz", - "integrity": "sha512-dD2ga5gLcQhsq1yNoQdy1MU4x4z7YnXM5bcG9SdQuiNr5KKuAmXixH1Mggwdah5o7EfholFbcNDPSVA6BIfaug==", - "requires": { - "async": "^2.6.2", - "binary": "~0.3.0", - "bl": "^2.2.0", - "buffer-crc32": "~0.2.5", - "buffermaker": "~1.2.0", - "debug": "^2.1.3", - "denque": "^1.3.0", - "lodash": "^4.17.4", - "minimatch": "^3.0.2", - "nested-error-stacks": "^2.0.0", - "optional": "^0.1.3", - "retry": "^0.10.1", - "snappy": "^6.0.1", - "uuid": "^3.0.0" - }, - "dependencies": { - "minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "retry": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz", - "integrity": "sha512-ZXUSQYTHdl3uS7IuCehYfMzKyIDBNoAuUblvy5oGO5UJSUTmStUUVPXbA9Qxd173Bgre53yCQczQuHgRWAdvJQ==" - } - } - }, - "kafkajs": { - "version": "2.2.4" - }, - "kafkajs-snappy": { - "version": "1.1.0", - "requires": { - "snappyjs": "^0.6.0" - } - }, - "kafkajs-snappy-typescript": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/kafkajs-snappy-typescript/-/kafkajs-snappy-typescript-1.0.3.tgz", - "integrity": "sha512-H5CiRKQ+RGJprWixvY5gDF3ofZEqQA0FFo0ePNgDCoJuH87n3gGUogtlUuei6A8c1ad/5qsQS3f9SSnmB/me4A==", - "requires": { - "snappy": "^7.2.2" - }, - "dependencies": { - "snappy": { - "version": "7.2.2", - "resolved": "https://registry.npmjs.org/snappy/-/snappy-7.2.2.tgz", - "integrity": "sha512-iADMq1kY0v3vJmGTuKcFWSXt15qYUz7wFkArOrsSg0IFfI3nJqIJvK2/ZbEIndg7erIJLtAVX2nSOqPz7DcwbA==", - "requires": { - "@napi-rs/snappy-android-arm-eabi": "7.2.2", - "@napi-rs/snappy-android-arm64": "7.2.2", - "@napi-rs/snappy-darwin-arm64": "7.2.2", - "@napi-rs/snappy-darwin-x64": "7.2.2", - "@napi-rs/snappy-freebsd-x64": "7.2.2", - "@napi-rs/snappy-linux-arm-gnueabihf": "7.2.2", - "@napi-rs/snappy-linux-arm64-gnu": "7.2.2", - "@napi-rs/snappy-linux-arm64-musl": "7.2.2", - "@napi-rs/snappy-linux-x64-gnu": "7.2.2", - "@napi-rs/snappy-linux-x64-musl": "7.2.2", - "@napi-rs/snappy-win32-arm64-msvc": "7.2.2", - "@napi-rs/snappy-win32-ia32-msvc": "7.2.2", - "@napi-rs/snappy-win32-x64-msvc": "7.2.2" - } - } - } - }, - "keyv": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.5.4.tgz", - "integrity": "sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==", - "dev": true, - "requires": { - "json-buffer": "3.0.1" - } - }, - "knex": { - "version": "2.4.2", - "requires": { - "colorette": "2.0.19", - "commander": "^9.1.0", - "debug": "4.3.4", - "escalade": "^3.1.1", - "esm": "^3.2.25", - "get-package-type": "^0.1.0", - "getopts": "2.3.0", - "interpret": "^2.2.0", - "lodash": "^4.17.21", - "pg-connection-string": "2.5.0", - "rechoir": "^0.8.0", - "resolve-from": "^5.0.0", - "tarn": "^3.0.2", - "tildify": "2.0.0" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2" - } - } - }, - "kuler": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", - "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==" - }, - "levn": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", - "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", - "dev": true, - "requires": { - "prelude-ls": "^1.2.1", - "type-check": "~0.4.0" - } - }, - "locate-path": { - "version": "6.0.0", - "dev": true, - "requires": { - "p-locate": "^5.0.0" - } - }, - "lodash": { - "version": "4.17.21" - }, - "lodash.difference": { - "version": "4.5.0", - "dev": true - }, - "lodash.find": { - "version": "4.6.0", - "dev": true - }, - "lodash.flattendeep": { - "version": "4.4.0", - "dev": true - }, - "lodash.get": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha512-z+Uw/vLuy6gQe8cfaFWD7p0wVv8fJl3mbzXh33RS+0oW2wvUqiRXiQ69gLWSLpgB5/6sU+r6BlQR0MBILadqTQ==", - "dev": true - }, - "lodash.intersection": { - "version": "4.4.0", - "dev": true - }, - "lodash.keyby": { - "version": "4.6.0", - "dev": true - }, - "lodash.merge": { - "version": "4.6.2", - "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", - "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", - "dev": true - }, - "log-symbols": { - "version": "4.1.0", - "dev": true, - "requires": { - "chalk": "^4.1.0", - "is-unicode-supported": "^0.1.0" - } - }, - "log4js": { - "version": "6.9.1", - "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.9.1.tgz", - "integrity": "sha512-1somDdy9sChrr9/f4UlzhdaGfDR2c/SaD2a4T7qEkG4jTS57/B3qmnjLYePwQ8cqWnUHZI0iAKxMBpCZICiZ2g==", - "requires": { - "date-format": "^4.0.14", - "debug": "^4.3.4", - "flatted": "^3.2.7", - "rfdc": "^1.3.0", - "streamroller": "^3.1.5" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, - "logform": { - "version": "1.10.0", - "resolved": "https://registry.npmjs.org/logform/-/logform-1.10.0.tgz", - "integrity": "sha512-em5ojIhU18fIMOw/333mD+ZLE2fis0EzXl1ZwHx4iQzmpQi6odNiY/t+ITNr33JZhT9/KEaH+UPIipr6a9EjWg==", - "requires": { - "colors": "^1.2.1", - "fast-safe-stringify": "^2.0.4", - "fecha": "^2.3.3", - "ms": "^2.1.1", - "triple-beam": "^1.2.0" - }, - "dependencies": { - "colors": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", - "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - } - } - }, - "long": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/long/-/long-1.1.2.tgz", - "integrity": "sha512-pjR3OP1X2VVQhCQlrq3s8UxugQsuoucwMOn9Yj/kN/61HMc+lDFJS5bvpNEHneZ9NVaSm8gNWxZvtGS7lqHb3Q==" - }, - "loupe": { - "version": "2.3.6", - "dev": true, - "requires": { - "get-func-name": "^2.0.0" - } - }, - "lru-cache": { - "version": "5.1.1", - "dev": true, - "requires": { - "yallist": "^3.0.2" - } - }, - "make-dir": { - "version": "3.1.0", - "dev": true, - "requires": { - "semver": "^7.5.3" - } - }, - "make-error": { - "version": "1.3.6", - "dev": true - }, - "media-typer": { - "version": "0.3.0" - }, - "merge-descriptors": { - "version": "1.0.1" - }, - "merge2": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", - "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", - "dev": true - }, - "methods": { - "version": "1.1.2" - }, - "micromatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", - "integrity": "sha512-LPP/3KorzCwBxfeUuZmaR6bG2kdeHSbe0P2tY3FLRU4vYrjYz5hI4QZwV0njUx3jeuKe67YukQ1LSPZBKDqO/Q==", - "dev": true, - "requires": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - } - }, - "mime": { - "version": "1.6.0" - }, - "mime-db": { - "version": "1.52.0" - }, - "mime-types": { - "version": "2.1.35", - "requires": { - "mime-db": "1.52.0" - } - }, - "mimic-response": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", - "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==", - "optional": true - }, - "minimatch": { - "version": "5.0.1", - "dev": true, - "requires": { - "brace-expansion": "^2.0.1" - }, - "dependencies": { - "brace-expansion": { - "version": "2.0.1", - "dev": true, - "requires": { - "balanced-match": "^1.0.0" - } - } - } - }, - "minimist": { - "version": "1.2.8", - "devOptional": true - }, - "mkdirp": { - "version": "0.5.6", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.6.tgz", - "integrity": "sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==", - "optional": true, - "requires": { - "minimist": "^1.2.6" - } - }, - "mocha": { - "version": "10.2.0", - "dev": true, - "requires": { - "ansi-colors": "4.1.1", - "browser-stdout": "1.3.1", - "chokidar": "3.5.3", - "debug": "4.3.4", - "diff": "5.0.0", - "escape-string-regexp": "4.0.0", - "find-up": "5.0.0", - "glob": "7.2.0", - "he": "1.2.0", - "js-yaml": "4.1.0", - "log-symbols": "4.1.0", - "minimatch": "5.0.1", - "ms": "2.1.3", - "nanoid": "3.3.3", - "serialize-javascript": "6.0.0", - "strip-json-comments": "3.1.1", - "supports-color": "8.1.1", - "workerpool": "6.2.1", - "yargs": "16.2.0", - "yargs-parser": "20.2.4", - "yargs-unparser": "2.0.0" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - }, - "dependencies": { - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "ms": { - "version": "2.1.3", - "dev": true - } - } - }, - "moment": { - "version": "2.29.4" - }, - "moment-timezone": { - "version": "0.5.45", - "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.45.tgz", - "integrity": "sha512-HIWmqA86KcmCAhnMAN0wuDOARV/525R2+lOLotuGFzn4HO+FH+/645z2wx0Dt3iDv6/p61SIvKnDstISainhLQ==", - "requires": { - "moment": "^2.29.4" - } - }, - "ms": { - "version": "2.0.0" - }, - "multiparty": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/multiparty/-/multiparty-4.2.1.tgz", - "integrity": "sha512-AvESCnNoQlZiOfP9R4mxN8M9csy2L16EIbWIkt3l4FuGti9kXBS8QVzlfyg4HEnarJhrzZilgNFlZtqmoiAIIA==", - "requires": { - "fd-slicer": "1.1.0", - "http-errors": "~1.7.0", - "safe-buffer": "5.1.2", - "uid-safe": "2.1.5" - }, - "dependencies": { - "depd": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz", - "integrity": "sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==" - }, - "http-errors": { - "version": "1.7.3", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.7.3.tgz", - "integrity": "sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw==", - "requires": { - "depd": "~1.1.2", - "inherits": "2.0.4", - "setprototypeof": "1.1.1", - "statuses": ">= 1.5.0 < 2", - "toidentifier": "1.0.0" - } - }, - "safe-buffer": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", - "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" - }, - "setprototypeof": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", - "integrity": "sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw==" - }, - "statuses": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", - "integrity": "sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==" - }, - "toidentifier": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", - "integrity": "sha512-yaOH/Pk/VEhBWWTlhI+qXxDFXlejDGcQipMlyxda9nthulaxLZUNcUqFxokp0vcYnvteJln5FNQDRrxj3YcbVw==" - } - } - }, - "nan": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.18.0.tgz", - "integrity": "sha512-W7tfG7vMOGtD30sHoZSSc/JVYiyDPEyQVso/Zz+/uQd0B0L46gtC+pHha5FFMRpil6fm/AoEcRWyOVi4+E/f8w==", - "optional": true - }, - "nanoid": { - "version": "3.3.3", - "dev": true - }, - "napi-build-utils": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", - "integrity": "sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==", - "optional": true - }, - "natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true - }, - "negotiator": { - "version": "0.6.3" - }, - "nested-error-stacks": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nested-error-stacks/-/nested-error-stacks-2.1.1.tgz", - "integrity": "sha512-9iN1ka/9zmX1ZvLV9ewJYEk9h7RyRRtqdK0woXcqohu8EWIerfPUjYJPg0ULy0UqP7cslmdGc8xKDJcojlKiaw==" - }, - "nise": { - "version": "5.1.9", - "resolved": "https://registry.npmjs.org/nise/-/nise-5.1.9.tgz", - "integrity": "sha512-qOnoujW4SV6e40dYxJOb3uvuoPHtmLzIk4TFo+j0jPJoC+5Z9xja5qH5JZobEPsa8+YYphMrOSwnrshEhG2qww==", - "dev": true, - "requires": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^11.2.2", - "@sinonjs/text-encoding": "^0.7.2", - "just-extend": "^6.2.0", - "path-to-regexp": "^6.2.1" - }, - "dependencies": { - "path-to-regexp": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.2.tgz", - "integrity": "sha512-GQX3SSMokngb36+whdpRXE+3f9V8UzyAorlYvOGx87ufGHehNTn5lCxrKtLyZ4Yl/wEKnNnr98ZzOwwDZV5ogw==", - "dev": true - } - } - }, - "nock": { - "version": "13.3.0", - "dev": true, - "requires": { - "debug": "^4.1.0", - "json-stringify-safe": "^5.0.1", - "lodash": "^4.17.21", - "propagate": "^2.0.0" - }, - "dependencies": { - "debug": { - "version": "4.3.4", - "dev": true, - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "dev": true - } - } - }, - "node-abi": { - "version": "2.30.1", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.30.1.tgz", - "integrity": "sha512-/2D0wOQPgaUWzVSVgRMx+trKJRC2UG4SUc4oCJoXx9Uxjtp0Vy3/kt7zcbxHF8+Z/pK3UloLWzBISg72brfy1w==", - "optional": true, - "requires": { - "semver": "^7.5.3" - } - }, - "node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "requires": { - "whatwg-url": "^5.0.0" - } - }, - "node-preload": { - "version": "0.2.1", - "dev": true, - "requires": { - "process-on-spawn": "^1.0.0" - } - }, - "node-releases": { - "version": "2.0.10", - "dev": true - }, - "node-sql-parser": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/node-sql-parser/-/node-sql-parser-5.2.0.tgz", - "integrity": "sha512-lO/9ox0fLl47ksqlwGrorXm8BD+5UbxvUhj6T1XiCXnvoVwWWn7gC2l396p9OXNii19dc4MkyTN3XeJGNUZ8jw==", - "requires": { - "@types/pegjs": "^0.10.0", - "big-integer": "^1.6.48" - } - }, - "nodemon": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.0.1.tgz", - "integrity": "sha512-g9AZ7HmkhQkqXkRc20w+ZfQ73cHLbE8hnPbtaFbFtCumZsjyMhKk9LajQ07U5Ux28lvFjZ5X7HvWR1xzU8jHVw==", - "dev": true, - "requires": { - "chokidar": "^3.5.2", - "debug": "^3.2.7", - "ignore-by-default": "^1.0.1", - "minimatch": "^3.1.2", - "pstree.remy": "^1.1.8", - "semver": "^7.5.3", - "simple-update-notifier": "^2.0.0", - "supports-color": "^5.5.0", - "touch": "^3.1.0", - "undefsafe": "^2.0.5" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - }, - "has-flag": { - "version": "3.0.0", - "dev": true - }, - "minimatch": { - "version": "3.1.2", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - }, - "ms": { - "version": "2.1.3", - "dev": true - }, - "supports-color": { - "version": "5.5.0", - "dev": true, - "requires": { - "has-flag": "^3.0.0" - } - } - } - }, - "noop-logger": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/noop-logger/-/noop-logger-0.1.1.tgz", - "integrity": "sha512-6kM8CLXvuW5crTxsAtva2YLrRrDaiTIkIePWs9moLHqbFWT94WpNFjwS/5dfLfECg5i/lkmw3aoqVidxt23TEQ==", - "optional": true - }, - "nopt": { - "version": "1.0.10", - "dev": true, - "requires": { - "abbrev": "1" - } - }, - "normalize-path": { - "version": "3.0.0", - "dev": true - }, - "npmlog": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", - "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", - "optional": true, - "requires": { - "are-we-there-yet": "~1.1.2", - "console-control-strings": "~1.1.0", - "gauge": "~2.7.3", - "set-blocking": "~2.0.0" - } - }, - "number-is-nan": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", - "integrity": "sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ==", - "optional": true - }, - "nyc": { - "version": "15.1.0", - "dev": true, - "requires": { - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "caching-transform": "^4.0.0", - "convert-source-map": "^1.7.0", - "decamelize": "^1.2.0", - "find-cache-dir": "^3.2.0", - "find-up": "^4.1.0", - "foreground-child": "^2.0.0", - "get-package-type": "^0.1.0", - "glob": "^7.1.6", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-hook": "^3.0.0", - "istanbul-lib-instrument": "^4.0.0", - "istanbul-lib-processinfo": "^2.0.2", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.0.2", - "make-dir": "^3.0.0", - "node-preload": "^0.2.1", - "p-map": "^3.0.0", - "process-on-spawn": "^1.0.0", - "resolve-from": "^5.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "spawn-wrap": "^2.0.0", - "test-exclude": "^6.0.0", - "yargs": "^15.0.2" - }, - "dependencies": { - "camelcase": { - "version": "5.3.1", - "dev": true - }, - "cliui": { - "version": "6.0.0", - "dev": true, - "requires": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.0", - "wrap-ansi": "^6.2.0" - } - }, - "decamelize": { - "version": "1.2.0", - "dev": true - }, - "find-up": { - "version": "4.1.0", - "dev": true, - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, - "locate-path": { - "version": "5.0.0", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } - }, - "p-limit": { - "version": "2.3.0", - "dev": true, - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "dev": true, - "requires": { - "p-limit": "^2.2.0" - } - }, - "wrap-ansi": { - "version": "6.2.0", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "y18n": { - "version": "4.0.3", - "dev": true - }, - "yargs": { - "version": "15.4.1", - "dev": true, - "requires": { - "cliui": "^6.0.0", - "decamelize": "^1.2.0", - "find-up": "^4.1.0", - "get-caller-file": "^2.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^2.0.0", - "set-blocking": "^2.0.0", - "string-width": "^4.2.0", - "which-module": "^2.0.0", - "y18n": "^4.0.0", - "yargs-parser": "^18.1.2" - } - }, - "yargs-parser": { - "version": "18.1.3", - "dev": true, - "requires": { - "camelcase": "^5.0.0", - "decamelize": "^1.2.0" - } - } - } - }, - "object-assign": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", - "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", - "optional": true - }, - "object-hash": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/object-hash/-/object-hash-2.2.0.tgz", - "integrity": "sha512-gScRMn0bS5fH+IuwyIFgnh9zBdo4DV+6GhygmWM9HyNJSgS0hScp1f5vjtm7oIIOiT9trXrShAkLFSc2IqKNgw==" - }, - "object-inspect": { - "version": "1.12.3" - }, - "on-finished": { - "version": "2.4.1", - "requires": { - "ee-first": "1.1.1" - } - }, - "on-headers": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/on-headers/-/on-headers-1.0.2.tgz", - "integrity": "sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==" - }, - "once": { - "version": "1.4.0", - "requires": { - "wrappy": "1" - } - }, - "one-time": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", - "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", - "requires": { - "fn.name": "1.x.x" - } - }, - "optional": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/optional/-/optional-0.1.4.tgz", - "integrity": "sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw==" - }, - "optionator": { - "version": "0.9.4", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", - "integrity": "sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==", - "dev": true, - "requires": { - "deep-is": "^0.1.3", - "fast-levenshtein": "^2.0.6", - "levn": "^0.4.1", - "prelude-ls": "^1.2.1", - "type-check": "^0.4.0", - "word-wrap": "^1.2.5" - } - }, - "os-homedir": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", - "integrity": "sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==", - "optional": true - }, - "p-limit": { - "version": "3.1.0", - "requires": { - "yocto-queue": "^0.1.0" - } - }, - "p-locate": { - "version": "5.0.0", - "dev": true, - "requires": { - "p-limit": "^3.0.2" - } - }, - "p-map": { - "version": "3.0.0", - "dev": true, - "requires": { - "aggregate-error": "^3.0.0" - } - }, - "p-try": { - "version": "2.2.0", - "dev": true - }, - "package-hash": { - "version": "4.0.0", - "dev": true, - "requires": { - "graceful-fs": "^4.1.15", - "hasha": "^5.0.0", - "lodash.flattendeep": "^4.4.0", - "release-zalgo": "^1.0.0" - } - }, - "parent-module": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", - "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", - "dev": true, - "requires": { - "callsites": "^3.0.0" - } - }, - "parseurl": { - "version": "1.3.3" - }, - "path-exists": { - "version": "4.0.0", - "dev": true - }, - "path-is-absolute": { - "version": "1.0.1", - "dev": true - }, - "path-key": { - "version": "3.1.1", - "dev": true - }, - "path-parse": { - "version": "1.0.7" - }, - "path-to-regexp": { - "version": "0.1.7" - }, - "path-type": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", - "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", - "dev": true - }, - "pathval": { - "version": "1.1.1", - "dev": true - }, - "pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==" - }, - "pg": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.12.0.tgz", - "integrity": "sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==", - "requires": { - "pg-cloudflare": "^1.1.1", - "pg-connection-string": "^2.6.4", - "pg-pool": "^3.6.2", - "pg-protocol": "^1.6.1", - "pg-types": "^2.1.0", - "pgpass": "1.x" - }, - "dependencies": { - "pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - } - } - }, - "pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", - "optional": true - }, - "pg-connection-string": { - "version": "2.5.0" - }, - "pg-hstore": { - "version": "2.3.4", - "resolved": "https://registry.npmjs.org/pg-hstore/-/pg-hstore-2.3.4.tgz", - "integrity": "sha512-N3SGs/Rf+xA1M2/n0JBiXFDVMzdekwLZLAO0g7mpDY9ouX+fDI7jS6kTq3JujmYbtNSJ53TJ0q4G98KVZSM4EA==", - "requires": { - "underscore": "^1.13.1" - } - }, - "pg-int8": { - "version": "1.0.1" - }, - "pg-pool": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", - "requires": {} - }, - "pg-protocol": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", - "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" - }, - "pg-types": { - "version": "2.2.0", - "requires": { - "pg-int8": "1.0.1", - "postgres-array": "~2.0.0", - "postgres-bytea": "~1.0.0", - "postgres-date": "~1.0.4", - "postgres-interval": "^1.1.0" - } - }, - "pgpass": { - "version": "1.0.5", - "requires": { - "split2": "^4.1.0" - } - }, - "picocolors": { - "version": "1.0.0", - "dev": true - }, - "picomatch": { - "version": "2.3.1", - "dev": true - }, - "pkg-dir": { - "version": "4.2.0", - "dev": true, - "requires": { - "find-up": "^4.0.0" - }, - "dependencies": { - "find-up": { - "version": "4.1.0", - "dev": true, - "requires": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - } - }, - "locate-path": { - "version": "5.0.0", - "dev": true, - "requires": { - "p-locate": "^4.1.0" - } - }, - "p-limit": { - "version": "2.3.0", - "dev": true, - "requires": { - "p-try": "^2.0.0" - } - }, - "p-locate": { - "version": "4.1.0", - "dev": true, - "requires": { - "p-limit": "^2.2.0" - } - } - } - }, - "postgres-array": { - "version": "2.0.0" - }, - "postgres-bytea": { - "version": "1.0.0" - }, - "postgres-date": { - "version": "1.0.7" - }, - "postgres-interval": { - "version": "1.2.0", - "requires": { - "xtend": "^4.0.0" - } - }, - "prebuild-install": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-5.3.0.tgz", - "integrity": "sha512-aaLVANlj4HgZweKttFNUVNRxDukytuIuxeK2boIMHjagNJCiVKWFsKF4tCE3ql3GbrD2tExPQ7/pwtEJcHNZeg==", - "optional": true, - "requires": { - "detect-libc": "^1.0.3", - "expand-template": "^2.0.3", - "github-from-package": "0.0.0", - "minimist": "^1.2.0", - "mkdirp": "^0.5.1", - "napi-build-utils": "^1.0.1", - "node-abi": "^2.7.0", - "noop-logger": "^0.1.1", - "npmlog": "^4.0.1", - "os-homedir": "^1.0.1", - "pump": "^2.0.1", - "rc": "^1.2.7", - "simple-get": "^2.7.0", - "tar-fs": "^1.13.0", - "tunnel-agent": "^0.6.0", - "which-pm-runs": "^1.0.0" - } - }, - "prelude-ls": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", - "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", - "dev": true - }, - "process-nextick-args": { - "version": "2.0.1" - }, - "process-on-spawn": { - "version": "1.0.0", - "dev": true, - "requires": { - "fromentries": "^1.2.0" - } - }, - "prom-client": { - "version": "14.2.0", - "resolved": "https://registry.npmjs.org/prom-client/-/prom-client-14.2.0.tgz", - "integrity": "sha512-sF308EhTenb/pDRPakm+WgiN+VdM/T1RaHj1x+MvAuT8UiQP8JmOEbxVqtkbfR4LrvOg5n7ic01kRBDGXjYikA==", - "requires": { - "tdigest": "^0.1.1" - } - }, - "propagate": { - "version": "2.0.1", - "dev": true - }, - "proxy-addr": { - "version": "2.0.7", - "requires": { - "forwarded": "0.2.0", - "ipaddr.js": "1.9.1" - } - }, - "proxy-from-env": { - "version": "1.1.0" - }, - "pstree.remy": { - "version": "1.1.8", - "dev": true - }, - "pump": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", - "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", - "optional": true, - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, - "punycode": { - "version": "2.3.0" - }, - "qs": { - "version": "6.11.0", - "requires": { - "side-channel": "^1.0.4" - } - }, - "querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==" - }, - "queue-microtask": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", - "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true - }, - "random-bytes": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/random-bytes/-/random-bytes-1.0.0.tgz", - "integrity": "sha512-iv7LhNVO047HzYR3InF6pUcUsPQiHTM1Qal51DcGSuZFBil1aBBWG5eHPNek7bvILMaYJ/8RU1e8w1AMdHmLQQ==" - }, - "randombytes": { - "version": "2.1.0", - "dev": true, - "requires": { - "safe-buffer": "^5.1.0" - } - }, - "range-parser": { - "version": "1.2.1" - }, - "raw-body": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", - "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", - "requires": { - "bytes": "3.1.2", - "http-errors": "2.0.0", - "iconv-lite": "0.4.24", - "unpipe": "1.0.0" - } - }, - "rc": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", - "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", - "optional": true, - "requires": { - "deep-extend": "^0.6.0", - "ini": "~1.3.0", - "minimist": "^1.2.0", - "strip-json-comments": "~2.0.1" - }, - "dependencies": { - "strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "optional": true - } - } - }, - "readable-stream": { - "version": "2.3.8", - "requires": { - "core-util-is": "~1.0.0", - "inherits": "~2.0.3", - "isarray": "~1.0.0", - "process-nextick-args": "~2.0.0", - "safe-buffer": "~5.1.1", - "string_decoder": "~1.1.1", - "util-deprecate": "~1.0.1" - }, - "dependencies": { - "safe-buffer": { - "version": "5.1.2" - } - } - }, - "readdirp": { - "version": "3.6.0", - "dev": true, - "requires": { - "picomatch": "^2.2.1" - } - }, - "rechoir": { - "version": "0.8.0", - "requires": { - "resolve": "^1.20.0" - } - }, - "release-zalgo": { - "version": "1.0.0", - "dev": true, - "requires": { - "es6-error": "^4.0.1" - } - }, - "require-directory": { - "version": "2.1.1", - "dev": true - }, - "require-from-string": { - "version": "2.0.2" - }, - "require-main-filename": { - "version": "2.0.0", - "dev": true - }, - "resolve": { - "version": "1.22.1", - "requires": { - "is-core-module": "^2.9.0", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - } - }, - "resolve-from": { - "version": "5.0.0" - }, - "retry": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", - "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==" - }, - "retry-as-promised": { - "version": "7.0.4", - "resolved": "https://registry.npmjs.org/retry-as-promised/-/retry-as-promised-7.0.4.tgz", - "integrity": "sha512-XgmCoxKWkDofwH8WddD0w85ZfqYz+ZHlr5yo+3YUCfycWawU56T5ckWXsScsj5B8tqUcIG67DxXByo3VUgiAdA==" - }, - "retry-request": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/retry-request/-/retry-request-7.0.2.tgz", - "integrity": "sha512-dUOvLMJ0/JJYEn8NrpOaGNE7X3vpI5XlZS/u0ANjqtcZVKnIxP7IgCFwrKTxENw29emmwug53awKtaMm4i9g5w==", - "requires": { - "@types/request": "^2.48.8", - "extend": "^3.0.2", - "teeny-request": "^9.0.0" - } - }, - "reusify": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", - "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==", - "dev": true - }, - "rfdc": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.1.tgz", - "integrity": "sha512-r5a3l5HzYlIC68TpmYKlxWjmOP6wiPJ1vWv2HeLhNsRZMrCkxeqxiHlQ21oXmQ4F3SiryXBHhAD7JZqvOJjFmg==" - }, - "rimraf": { - "version": "3.0.2", - "dev": true, - "requires": { - "glob": "^7.1.3" - } - }, - "run-parallel": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", - "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, - "requires": { - "queue-microtask": "^1.2.2" - } - }, - "safe-buffer": { - "version": "5.2.1" - }, - "safe-stable-stringify": { - "version": "2.4.3", - "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", - "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==" - }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" - }, - "sax": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", - "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" - }, - "semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "requires": { - "lru-cache": "^6.0.0" - }, - "dependencies": { - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "requires": { - "yallist": "^4.0.0" - } - }, - "yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - } - } - }, - "send": { - "version": "0.18.0", - "requires": { - "debug": "2.6.9", - "depd": "2.0.0", - "destroy": "1.2.0", - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "etag": "~1.8.1", - "fresh": "0.5.2", - "http-errors": "2.0.0", - "mime": "1.6.0", - "ms": "2.1.3", - "on-finished": "2.4.1", - "range-parser": "~1.2.1", - "statuses": "2.0.1" - }, - "dependencies": { - "ms": { - "version": "2.1.3" - } - } - }, - "sequelize": { - "version": "6.37.3", - "resolved": "https://registry.npmjs.org/sequelize/-/sequelize-6.37.3.tgz", - "integrity": "sha512-V2FTqYpdZjPy3VQrZvjTPnOoLm0KudCRXfGWp48QwhyPPp2yW8z0p0sCYZd/em847Tl2dVxJJ1DR+hF+O77T7A==", - "requires": { - "@types/debug": "^4.1.8", - "@types/validator": "^13.7.17", - "debug": "^4.3.4", - "dottie": "^2.0.6", - "inflection": "^1.13.4", - "lodash": "^4.17.21", - "moment": "^2.29.4", - "moment-timezone": "^0.5.43", - "pg-connection-string": "^2.6.1", - "retry-as-promised": "^7.0.4", - "semver": "^7.5.3", - "sequelize-pool": "^7.1.0", - "toposort-class": "^1.0.1", - "uuid": "^8.3.2", - "validator": "^13.9.0", - "wkx": "^0.5.0" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" - }, - "uuid": { - "version": "8.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", - "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" - } - } - }, - "sequelize-pool": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/sequelize-pool/-/sequelize-pool-7.1.0.tgz", - "integrity": "sha512-G9c0qlIWQSK29pR/5U2JF5dDQeqqHRragoyahj/Nx4KOOQ3CPPfzxnfqFPCSB7x5UgjOgnZ61nSxz+fjDpRlJg==" - }, - "serialize-javascript": { - "version": "6.0.0", - "dev": true, - "requires": { - "randombytes": "^2.1.0" - } - }, - "serve-static": { - "version": "1.15.0", - "requires": { - "encodeurl": "~1.0.2", - "escape-html": "~1.0.3", - "parseurl": "~1.3.3", - "send": "0.18.0" - } - }, - "set-blocking": { - "version": "2.0.0", - "devOptional": true - }, - "setprototypeof": { - "version": "1.2.0" - }, - "shebang-command": { - "version": "2.0.0", - "dev": true, - "requires": { - "shebang-regex": "^3.0.0" - } - }, - "shebang-regex": { - "version": "3.0.0", - "dev": true - }, - "side-channel": { - "version": "1.0.4", - "requires": { - "call-bind": "^1.0.0", - "get-intrinsic": "^1.0.2", - "object-inspect": "^1.9.0" - } - }, - "signal-exit": { - "version": "3.0.7", - "devOptional": true - }, - "simple-concat": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", - "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", - "optional": true - }, - "simple-get": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-2.8.2.tgz", - "integrity": "sha512-Ijd/rV5o+mSBBs4F/x9oDPtTx9Zb6X9brmnXvMW4J7IR15ngi9q5xxqWBKU744jTZiaXtxaPL7uHG6vtN8kUkw==", - "optional": true, - "requires": { - "decompress-response": "^3.3.0", - "once": "^1.3.1", - "simple-concat": "^1.0.0" - } - }, - "simple-swizzle": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", - "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", - "requires": { - "is-arrayish": "^0.3.1" - } - }, - "simple-update-notifier": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", - "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", - "dev": true, - "requires": { - "semver": "^7.5.3" - } - }, - "sinon": { - "version": "17.0.1", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-17.0.1.tgz", - "integrity": "sha512-wmwE19Lie0MLT+ZYNpDymasPHUKTaZHUH/pKEubRXIzySv9Atnlw+BUMGCzWgV7b7wO+Hw6f1TEOr0IUnmU8/g==", - "dev": true, - "requires": { - "@sinonjs/commons": "^3.0.0", - "@sinonjs/fake-timers": "^11.2.2", - "@sinonjs/samsam": "^8.0.0", - "diff": "^5.1.0", - "nise": "^5.1.5", - "supports-color": "^7.2.0" - }, - "dependencies": { - "diff": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-5.2.0.tgz", - "integrity": "sha512-uIFDxqpRZGZ6ThOk84hEfqWoHx2devRFvpTZcTHur85vImfaxUbTW9Ryh4CpCuDnToOP1CEtXKIgytHBPVff5A==", - "dev": true - }, - "supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, - "slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true - }, - "slug": { - "version": "9.1.0", - "resolved": "https://registry.npmjs.org/slug/-/slug-9.1.0.tgz", - "integrity": "sha512-ioOsCfzQSu+D6NZ8XMCR8IW9FgvF8W7Xzz56hBkB/ALvNaWeBs2MUvvPugq3GCrxfHPFeK6hAxGkY/WLnfX2Lg==" - }, - "snappy": { - "version": "6.3.5", - "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.3.5.tgz", - "integrity": "sha512-lonrUtdp1b1uDn1dbwgQbBsb5BbaiLeKq+AGwOk2No+en+VvJThwmtztwulEQsLinRF681pBqib0NUZaizKLIA==", - "optional": true, - "requires": { - "bindings": "^1.3.1", - "nan": "^2.14.1", - "prebuild-install": "5.3.0" - } - }, - "snappyjs": { - "version": "0.6.1" - }, - "source-map": { - "version": "0.6.1", - "dev": true - }, - "spawn-wrap": { - "version": "2.0.0", - "dev": true, - "requires": { - "foreground-child": "^2.0.0", - "is-windows": "^1.0.2", - "make-dir": "^3.0.0", - "rimraf": "^3.0.0", - "signal-exit": "^3.0.2", - "which": "^2.0.1" - } - }, - "split2": { - "version": "4.1.0" - }, - "stack-trace": { - "version": "0.0.10", - "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", - "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==" - }, - "statuses": { - "version": "2.0.1" - }, - "stream-browserify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", - "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==", - "requires": { - "inherits": "~2.0.4", - "readable-stream": "^3.5.0" - }, - "dependencies": { - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, - "stream-events": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/stream-events/-/stream-events-1.0.5.tgz", - "integrity": "sha512-E1GUzBSgvct8Jsb3v2X15pjzN1tYebtbLaMg+eBOUOAxgbLoSbT2NS91ckc5lJD1KfLjId+jXJRgo0qnV5Nerg==", - "requires": { - "stubs": "^3.0.0" - } - }, - "stream-shift": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", - "integrity": "sha512-76ORR0DO1o1hlKwTbi/DM3EXWGf3ZJYO8cXX5RJwnul2DEg2oyoZyjLNoQM8WsvZiFKCRfC1O0J7iCvie3RZmQ==" - }, - "streamroller": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-3.1.5.tgz", - "integrity": "sha512-KFxaM7XT+irxvdqSP1LGLgNWbYN7ay5owZ3r/8t77p+EtSUAfUgtl7be3xtqtOmGUl9K9YPO2ca8133RlTjvKw==", - "requires": { - "date-format": "^4.0.14", - "debug": "^4.3.4", - "fs-extra": "^8.1.0" - }, - "dependencies": { - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, - "string_decoder": { - "version": "1.1.1", - "requires": { - "safe-buffer": "~5.1.0" - }, - "dependencies": { - "safe-buffer": { - "version": "5.1.2" - } - } - }, - "string-width": { - "version": "4.2.3", - "devOptional": true, - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, - "strip-ansi": { - "version": "6.0.1", - "devOptional": true, - "requires": { - "ansi-regex": "^5.0.1" - } - }, - "strip-bom": { - "version": "4.0.0", - "dev": true - }, - "strip-json-comments": { - "version": "3.1.1", - "dev": true - }, - "strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" - }, - "stubs": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stubs/-/stubs-3.0.0.tgz", - "integrity": "sha512-PdHt7hHUJKxvTCgbKX9C1V/ftOcjJQgz8BZwNfV5c4B6dcGqlpelTbJ999jBGZ2jYiPAwcX5dP6oBwVlBlUbxw==" - }, - "superagent": { - "version": "3.8.3", - "dev": true, - "requires": { - "component-emitter": "^1.2.0", - "cookiejar": "^2.1.0", - "debug": "^3.1.0", - "extend": "^3.0.0", - "form-data": "^2.3.1", - "formidable": "^1.2.0", - "methods": "^1.1.1", - "mime": "^1.4.1", - "qs": "^6.5.1", - "readable-stream": "^2.3.5" - }, - "dependencies": { - "debug": { - "version": "3.2.7", - "dev": true, - "requires": { - "ms": "^2.1.1" - } - }, - "form-data": { - "version": "2.5.1", - "dev": true, - "requires": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.6", - "mime-types": "^2.1.12" - } - }, - "ms": { - "version": "2.1.3", - "dev": true - } - } - }, - "supports-color": { - "version": "8.1.1", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - }, - "supports-preserve-symlinks-flag": { - "version": "1.0.0" - }, - "tar-fs": { - "version": "1.16.3", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-1.16.3.tgz", - "integrity": "sha512-NvCeXpYx7OsmOh8zIOP/ebG55zZmxLE0etfWRbWok+q2Qo8x/vOR/IJT1taADXPe+jsiu9axDb3X4B+iIgNlKw==", - "optional": true, - "requires": { - "chownr": "^1.0.1", - "mkdirp": "^0.5.1", - "pump": "^1.0.0", - "tar-stream": "^1.1.2" - }, - "dependencies": { - "pump": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/pump/-/pump-1.0.3.tgz", - "integrity": "sha512-8k0JupWme55+9tCVE+FS5ULT3K6AbgqrGa58lTT49RpyfwwcGedHqaC5LlQNdEAumn/wFsu6aPwkuPMioy8kqw==", - "optional": true, - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - } - } - }, - "tar-stream": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-1.6.2.tgz", - "integrity": "sha512-rzS0heiNf8Xn7/mpdSVVSMAWAoy9bfb1WOTYC78Z0UQKeKa/CWS8FOq0lKGNa8DWKAn9gxjCvMLYc5PGXYlK2A==", - "optional": true, - "requires": { - "bl": "^1.0.0", - "buffer-alloc": "^1.2.0", - "end-of-stream": "^1.0.0", - "fs-constants": "^1.0.0", - "readable-stream": "^2.3.0", - "to-buffer": "^1.1.1", - "xtend": "^4.0.0" - }, - "dependencies": { - "bl": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-1.2.3.tgz", - "integrity": "sha512-pvcNpa0UU69UT341rO6AYy4FVAIkUHuZXRIWbq+zHnsVcRzDDjIAhGuuYoi0d//cwIwtt4pkpKycWEfjdV+vww==", - "optional": true, - "requires": { - "readable-stream": "^2.3.5", - "safe-buffer": "^5.1.1" - } - } - } - }, - "tarn": { - "version": "3.0.2" - }, - "tdigest": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.2.tgz", - "integrity": "sha512-+G0LLgjjo9BZX2MfdvPfH+MKLCrxlXSYec5DaPYP1fe6Iyhf0/fSmJ0bFiZ1F8BT6cGXl2LpltQptzjXKWEkKA==", - "requires": { - "bintrees": "1.0.2" - } - }, - "teeny-request": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/teeny-request/-/teeny-request-9.0.0.tgz", - "integrity": "sha512-resvxdc6Mgb7YEThw6G6bExlXKkv6+YbuzGg9xuXxSgxJF7Ozs+o8Y9+2R3sArdWdW8nOokoQb1yrpFB0pQK2g==", - "requires": { - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.0", - "node-fetch": "^2.6.9", - "stream-events": "^1.0.5", - "uuid": "^9.0.0" - }, - "dependencies": { - "agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "requires": { - "debug": "4" - } - }, - "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==", - "requires": { - "ms": "2.1.2" - } - }, - "http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "requires": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - } - }, - "https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "requires": { - "agent-base": "6", - "debug": "4" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - }, - "uuid": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", - "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==" - } - } - }, - "test-exclude": { - "version": "6.0.0", - "dev": true, - "requires": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "dependencies": { - "minimatch": { - "version": "3.1.2", - "dev": true, - "requires": { - "brace-expansion": "^1.1.7" - } - } - } - }, - "text-hex": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", - "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==" - }, - "text-table": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", - "integrity": "sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw==", - "dev": true - }, - "tildify": { - "version": "2.0.0" - }, - "to-buffer": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/to-buffer/-/to-buffer-1.1.1.tgz", - "integrity": "sha512-lx9B5iv7msuFYE3dytT+KE5tap+rNYw+K4jVkb9R/asAb+pbBSM17jtunHplhBe6RRJdZx3Pn2Jph24O32mOVg==", - "optional": true - }, - "to-fast-properties": { - "version": "2.0.0", - "dev": true - }, - "to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "requires": { - "is-number": "^7.0.0" - } - }, - "toidentifier": { - "version": "1.0.1" - }, - "toposort-class": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz", - "integrity": "sha512-OsLcGGbYF3rMjPUf8oKktyvCiUxSbqMMS39m33MAjLTC1DVIH6x3WSt63/M77ihI09+Sdfk1AXvfhCEeUmC7mg==" - }, - "touch": { - "version": "3.1.0", - "dev": true, - "requires": { - "nopt": "~1.0.10" - } - }, - "tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" - }, - "traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha512-iawgk0hLP3SxGKDfnDJf8wTz4p2qImnyihM5Hh/sGvQ3K37dPi/w8sRhdNIxYA1TwFwc5mDhIJq+O0RsvXBKdQ==" - }, - "trino-client": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/trino-client/-/trino-client-0.2.2.tgz", - "integrity": "sha512-TMndAbFiiGlAbJotsqsbMSqFZ8sPtE+KBIl/qYEAI9+eAXruNvz0BhxFC67PE39eouVn+JtF6L8Yr2q6/sAlAA==", - "requires": { - "axios": "1.6.2" - } - }, - "triple-beam": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", - "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==" - }, - "ts-api-utils": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/ts-api-utils/-/ts-api-utils-1.3.0.tgz", - "integrity": "sha512-UQMIo7pb8WRomKR1/+MFVLTroIvDVtMX3K6OUir8ynLyzB8Jeriont2bTAtmNPa1ekAgN7YPDyf6V+ygrdU+eQ==", - "dev": true, - "requires": {} - }, - "ts-node": { - "version": "10.9.1", - "dev": true, - "requires": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "dependencies": { - "diff": { - "version": "4.0.2", - "dev": true - } - } - }, - "tsconfig-paths": { - "version": "4.1.2", - "dev": true, - "requires": { - "json5": "^2.2.2", - "minimist": "^1.2.6", - "strip-bom": "^3.0.0" - }, - "dependencies": { - "strip-bom": { - "version": "3.0.0", - "dev": true - } - } - }, - "tslib": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.2.tgz", - "integrity": "sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==" - }, - "tunnel-agent": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", - "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", - "optional": true, - "requires": { - "safe-buffer": "^5.0.1" - } - }, - "type-check": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", - "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", - "dev": true, - "requires": { - "prelude-ls": "^1.2.1" - } - }, - "type-detect": { - "version": "4.0.8", - "dev": true - }, - "type-fest": { - "version": "0.8.1", - "dev": true - }, - "type-is": { - "version": "1.6.18", - "requires": { - "media-typer": "0.3.0", - "mime-types": "~2.1.24" - } - }, - "typedarray-to-buffer": { - "version": "3.1.5", - "dev": true, - "requires": { - "is-typedarray": "^1.0.0" - } - }, - "typescript": { - "version": "4.9.5", - "dev": true - }, - "uid-safe": { - "version": "2.1.5", - "resolved": "https://registry.npmjs.org/uid-safe/-/uid-safe-2.1.5.tgz", - "integrity": "sha512-KPHm4VL5dDXKz01UuEd88Df+KzynaohSL9fBh096KWAxSKZQDI2uBrVqtvRM4rwrIrRRKsdLNML/lnaaVSRioA==", - "requires": { - "random-bytes": "~1.0.0" - } - }, - "undefsafe": { - "version": "2.0.5", - "dev": true - }, - "underscore": { - "version": "1.13.6", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", - "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" - }, - "universalify": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", - "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" - }, - "unpipe": { - "version": "1.0.0" - }, - "update-browserslist-db": { - "version": "1.0.10", - "dev": true, - "requires": { - "escalade": "^3.1.1", - "picocolors": "^1.0.0" - } - }, - "uri-js": { - "version": "4.4.1", - "requires": { - "punycode": "^2.1.0" - } - }, - "url": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", - "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", - "requires": { - "punycode": "1.3.2", - "querystring": "0.2.0" - }, - "dependencies": { - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" - } - } - }, - "util": { - "version": "0.12.5", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", - "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", - "requires": { - "inherits": "^2.0.3", - "is-arguments": "^1.0.4", - "is-generator-function": "^1.0.7", - "is-typed-array": "^1.1.3", - "which-typed-array": "^1.1.2" - } - }, - "util-deprecate": { - "version": "1.0.2" - }, - "utils-merge": { - "version": "1.0.1" - }, - "uuid": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.1.0.tgz", - "integrity": "sha512-DIWtzUkw04M4k3bf1IcpS2tngXEL26YUD2M0tMDUpnUrz2hgzUBlD55a4FjdLGPvfHxS6uluGWvaVEqgBcVa+g==" - }, - "v8-compile-cache-lib": { - "version": "3.0.1", - "dev": true - }, - "validator": { - "version": "13.12.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.12.0.tgz", - "integrity": "sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==" - }, - "vary": { - "version": "1.1.2" - }, - "webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" - }, - "whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "requires": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, - "which": { - "version": "2.0.2", - "dev": true, - "requires": { - "isexe": "^2.0.0" - } - }, - "which-module": { - "version": "2.0.0", - "dev": true - }, - "which-pm-runs": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/which-pm-runs/-/which-pm-runs-1.1.0.tgz", - "integrity": "sha512-n1brCuqClxfFfq/Rb0ICg9giSZqCS+pLtccdag6C2HyufBrh3fBOiy9nb6ggRMvWOVH5GrdJskj5iGTZNxd7SA==", - "optional": true - }, - "which-typed-array": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", - "integrity": "sha512-qe9UWWpkeG5yzZ0tNYxDmd7vo58HDBc39mZ0xWWpolAGADdFOzkfamWLDxkOWcvHQKVmdTyQdLD4NOfjLWTKew==", - "requires": { - "available-typed-arrays": "^1.0.5", - "call-bind": "^1.0.2", - "for-each": "^0.3.3", - "gopd": "^1.0.1", - "has-tostringtag": "^1.0.0" - } - }, - "wide-align": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.5.tgz", - "integrity": "sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==", - "optional": true, - "requires": { - "string-width": "^1.0.2 || 2 || 3 || 4" - } - }, - "winston": { - "version": "2.4.7", - "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.7.tgz", - "integrity": "sha512-vLB4BqzCKDnnZH9PHGoS2ycawueX4HLqENXQitvFHczhgW2vFpSOn31LZtVr1KU8YTw7DS4tM+cqyovxo8taVg==", - "requires": { - "async": "^2.6.4", - "colors": "1.0.x", - "cycle": "1.0.x", - "eyes": "0.1.x", - "isstream": "0.1.x", - "stack-trace": "0.0.x" - } - }, - "winston-compat": { - "version": "0.1.5", - "resolved": "https://registry.npmjs.org/winston-compat/-/winston-compat-0.1.5.tgz", - "integrity": "sha512-EPvPcHT604AV3Ji6d3+vX8ENKIml9VSxMRnPQ+cuK/FX6f3hvPP2hxyoeeCOCFvDrJEujalfcKWlWPvAnFyS9g==", - "requires": { - "cycle": "~1.0.3", - "logform": "^1.6.0", - "triple-beam": "^1.2.0" - } - }, - "winston-daily-rotate-file": { - "version": "3.2.3", - "resolved": "https://registry.npmjs.org/winston-daily-rotate-file/-/winston-daily-rotate-file-3.2.3.tgz", - "integrity": "sha512-BOvmvQH2WaiexOjzj14YNHSc18IDyZJ9t4pMsbTERjpjMltoBVijM8DDJJPr2jSqELSNnbgGPBk3kDQSRgOAtQ==", - "requires": { - "file-stream-rotator": "^0.2.1", - "semver": "^7.5.3", - "triple-beam": "^1.3.0", - "winston-compat": "^0.1.4", - "winston-transport": "^4.2.0" - } - }, - "winston-transport": { - "version": "4.7.0", - "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.7.0.tgz", - "integrity": "sha512-ajBj65K5I7denzer2IYW6+2bNIVqLGDHqDw3Ow8Ohh+vdW+rv4MZ6eiDvHoKhfJFZ2auyN8byXieDDJ96ViONg==", - "requires": { - "logform": "^2.3.2", - "readable-stream": "^3.6.0", - "triple-beam": "^1.3.0" - }, - "dependencies": { - "@colors/colors": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.5.0.tgz", - "integrity": "sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ==" - }, - "fecha": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", - "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==" - }, - "logform": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/logform/-/logform-2.5.1.tgz", - "integrity": "sha512-9FyqAm9o9NKKfiAKfZoYo9bGXXuwMkxQiQttkT4YjjVtQVIQtK6LmVtlxmCaFswo6N4AfEkHqZTV0taDtPotNg==", - "requires": { - "@colors/colors": "1.5.0", - "@types/triple-beam": "^1.3.2", - "fecha": "^4.2.0", - "ms": "^2.1.1", - "safe-stable-stringify": "^2.3.1", - "triple-beam": "^1.3.0" - } - }, - "ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" - }, - "readable-stream": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", - "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", - "requires": { - "inherits": "^2.0.3", - "string_decoder": "^1.1.1", - "util-deprecate": "^1.0.1" - } - } - } - }, - "wkx": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/wkx/-/wkx-0.5.0.tgz", - "integrity": "sha512-Xng/d4Ichh8uN4l0FToV/258EjMGU9MGcA0HV2d9B/ZpZB3lqQm7nkOdZdm5GhKtLLhAE7PiVQwN4eN+2YJJUg==", - "requires": { - "@types/node": "*" - } - }, - "word-wrap": { - "version": "1.2.5", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", - "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", - "dev": true - }, - "workerpool": { - "version": "6.2.1", - "dev": true - }, - "wrap-ansi": { - "version": "7.0.0", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, - "wrappy": { - "version": "1.0.2" - }, - "write-file-atomic": { - "version": "3.0.3", - "dev": true, - "requires": { - "imurmurhash": "^0.1.4", - "is-typedarray": "^1.0.0", - "signal-exit": "^3.0.2", - "typedarray-to-buffer": "^3.1.5" - } - }, - "xml2js": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.5.0.tgz", - "integrity": "sha512-drPFnkQJik/O+uPKpqSgr22mpuFHqKdbS835iAQrUC73L2F5WkboIRd63ai/2Yg6I1jzifPFKH2NTK+cfglkIA==", - "requires": { - "sax": ">=0.6.0", - "xmlbuilder": "~11.0.0" - } - }, - "xmlbuilder": { - "version": "11.0.1", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", - "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" - }, - "xtend": { - "version": "4.0.2" - }, - "y18n": { - "version": "5.0.8", - "dev": true - }, - "yallist": { - "version": "3.1.1", - "dev": true - }, - "yargs": { - "version": "16.2.0", - "dev": true, - "requires": { - "cliui": "^7.0.2", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.0", - "y18n": "^5.0.5", - "yargs-parser": "^20.2.2" - } - }, - "yargs-parser": { - "version": "20.2.4", - "dev": true - }, - "yargs-unparser": { - "version": "2.0.0", - "dev": true, - "requires": { - "camelcase": "^6.0.0", - "decamelize": "^4.0.0", - "flat": "^5.0.2", - "is-plain-obj": "^2.1.0" - } - }, - "yn": { - "version": "3.1.1", - "dev": true - }, - "yocto-queue": { - "version": "0.1.0" - } - } -} diff --git a/api-service/src/v1/helpers/Datasets.ts b/api-service/src/v1/helpers/Datasets.ts index 94a12771..07d4daa3 100644 --- a/api-service/src/v1/helpers/Datasets.ts +++ b/api-service/src/v1/helpers/Datasets.ts @@ -24,6 +24,8 @@ export class Datasets { private created_by: string private updated_by: string private published_date: Date + private api_version: string + private version_key: string constructor(payload: any) { if (payload.id) { this.id = payload.id @@ -46,11 +48,13 @@ export class Datasets { this.created_by = payload.created_by this.updated_by = payload.updated_by this.published_date = payload.published_date + this.api_version = "v1" + this.version_key = payload.version_key || "1" } public getValues() { this.validateDenormConfig(); - return Object.assign(this.removeNullValues({ id: this.id, dataset_id: this.dataset_id, type: this.type, name: this.name, validation_config: this.validation_config, extraction_config: this.extraction_config, dedup_config: this.dedup_config, data_schema: this.data_schema, router_config: this.router_config, denorm_config: this.denorm_config, dataset_config: this.dataset_config, tags: this.tags, status: this.status, created_by: this.created_by, updated_by: this.updated_by, published_date: this.published_date }), { "updated_date": new Date }) + return Object.assign(this.removeNullValues({ id: this.id, dataset_id: this.dataset_id, type: this.type, name: this.name, validation_config: this.validation_config, extraction_config: this.extraction_config, dedup_config: this.dedup_config, data_schema: this.data_schema, router_config: this.router_config, denorm_config: this.denorm_config, dataset_config: this.dataset_config, tags: this.tags, status: this.status, api_version: this.api_version, version_key: this.version_key, created_by: this.created_by, updated_by: this.updated_by, published_date: this.published_date }), { "updated_date": new Date }) } public setValues() { diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index b2248e1d..07582f41 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -171,7 +171,28 @@ "minLength": 1 }, "transformation_function": { - "type": "object" + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + }, + "condition": { + "type": "string" + }, + "dataType": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "type", + "expr" + ] }, "mode": { "type": "string", From e06dd058e096d9b63c25e3e07c563918b3e7e532 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Tue, 25 Jun 2024 15:49:57 +0530 Subject: [PATCH 002/235] Issue OBS-I1 feat: initial routes and files added --- .../DatasetHealthValidationSchema.json | 2 +- .../controllers/DatasetReset/DatasetReset.ts | 72 +++++++++++++++++++ .../DatasetResetValidationSchema.json | 38 ++++++++++ api-service/src/v2/routes/Router.ts | 2 + 4 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 api-service/src/v2/controllers/DatasetReset/DatasetReset.ts create mode 100644 api-service/src/v2/controllers/DatasetReset/DatasetResetValidationSchema.json diff --git a/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json b/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json index 152b55c7..5a788259 100644 --- a/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json @@ -25,7 +25,7 @@ "request": { "type": "object", "properties": { - "datasets": { + "dataset": { "type": "string", "minLength": 1 }, diff --git a/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts b/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts new file mode 100644 index 00000000..cf56cf65 --- /dev/null +++ b/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts @@ -0,0 +1,72 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { schemaValidation } from "../../services/ValidationService"; +import DatasetResetRequestSchema from "./DatasetResetValidationSchema.json" +import { ErrorObject } from "../../types/ResponseModel"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { DatasetStatus, HealthStatus } from "../../types/DatasetModels"; +import { Dataset } from "../../models/Dataset"; +import logger from "../../logger"; +import { getInfraHealth, getProcessingHealth, getQueryHealth } from "../../services/HealthService"; +import { Datasource } from "../../models/Datasource"; + +export const apiId = "api.dataset.reset"; +export const errorCode = "DATASET_RESET_FAILURE" + + + +const datasetReset = async (req: Request, res: Response) => { + const resmsgid = _.get(res, "resmsgid"); + const requestBody = req.body; + const datasetId = _.get(req, "params.datasetId") + const msgid = _.get(req, ["body", "params", "msgid"]); + try { + const isRequestValid: Record = schemaValidation(req.body, DatasetResetRequestSchema) + if (!isRequestValid.isValid) { + const code = "DATASET_RESET_INPUT_INVALID" + logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) + return ResponseHandler.errorResponse({ + code, + message: isRequestValid.message, + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + } + + const dataset = await getLiveDatasets(datasetId) + if(_.isEmpty(dataset)) { + const code = "DATASET_RESET_NO_DATASET" + const message = `There are no live dataset exists with given dataset_id: ${datasetId}` + logger.error({ code, apiId, msgid, requestBody, resmsgid, message: message }) + return ResponseHandler.errorResponse({ + code, + message, + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + } + logger.debug(apiId, msgid, resmsgid, "dataset", dataset) + + + } catch (error: any) { + logger.error({ ...error, apiId, code: errorCode, msgid, requestBody, resmsgid }); + let errorMessage = error; + const statusCode = _.get(error, "statusCode") + if (!statusCode || statusCode == 500) { + errorMessage = { code: errorCode, message: "Failed to reset the dataset" } + } + ResponseHandler.errorResponse(errorMessage, req, res); + } + + + +} +const getLiveDatasets = async (ids: string): Promise> => { + return Dataset.findAll({ attributes: ['dataset_id', 'status', 'type'], where: { dataset_id: ids, status: DatasetStatus.Live }, raw: true }); +} + +const getDataSources = async (ids: Record): Promise> => { + return Datasource.findAll({ attributes: ['dataset_id', 'datasource'], where: { dataset_id: ids }, raw: true }); +} + +export default datasetReset; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetReset/DatasetResetValidationSchema.json b/api-service/src/v2/controllers/DatasetReset/DatasetResetValidationSchema.json new file mode 100644 index 00000000..9450c235 --- /dev/null +++ b/api-service/src/v2/controllers/DatasetReset/DatasetResetValidationSchema.json @@ -0,0 +1,38 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": [ + "msgid" + ], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "category": { + "type": "string", + "enum": ["processing", "query"] + } + }, + "required": [ + "category" + ] + } + } +} \ No newline at end of file diff --git a/api-service/src/v2/routes/Router.ts b/api-service/src/v2/routes/Router.ts index fb714a97..3af6f314 100644 --- a/api-service/src/v2/routes/Router.ts +++ b/api-service/src/v2/routes/Router.ts @@ -21,6 +21,7 @@ import GenerateSignedURL from "../controllers/GenerateSignedURL/GenerateSignedUR import { sqlQuery } from "../controllers/QueryWrapper/SqlQueryWrapper"; import DatasetStatusTansition from "../controllers/DatasetStatusTransition/DatasetStatusTransition"; import datasetHealth from "../controllers/DatasetHealth/DatasetHealth"; +import datasetReset from "../controllers/DatasetReset/DatasetReset"; export const router = express.Router(); @@ -41,6 +42,7 @@ router.post("/template/query/:templateId", setDataToRequestObject("api.query.tem router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); +router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); From 6503e35be657940b4e7c4c868a7916228deb7bdc Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 26 Jun 2024 13:07:07 +0530 Subject: [PATCH 003/235] Issue #OBS-I2 feat: added query action --- .../controllers/DatasetReset/DatasetReset.ts | 39 +++++++++++++++---- .../DatasetStatusTransition.ts | 2 +- api-service/src/v2/services/HealthService.ts | 39 ++++++++++++++++--- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts b/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts index cf56cf65..7462b6f7 100644 --- a/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts @@ -4,11 +4,12 @@ import { schemaValidation } from "../../services/ValidationService"; import DatasetResetRequestSchema from "./DatasetResetValidationSchema.json" import { ErrorObject } from "../../types/ResponseModel"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { DatasetStatus, HealthStatus } from "../../types/DatasetModels"; +import { DatasetStatus, DatasetType, HealthStatus } from "../../types/DatasetModels"; import { Dataset } from "../../models/Dataset"; import logger from "../../logger"; -import { getInfraHealth, getProcessingHealth, getQueryHealth } from "../../services/HealthService"; +import { getDruidIndexers, getFlinkHealthStatus, restartDruidIndexers } from "../../services/HealthService"; import { Datasource } from "../../models/Datasource"; +import { restartPipeline } from "../DatasetStatusTransition/DatasetStatusTransition"; export const apiId = "api.dataset.reset"; export const errorCode = "DATASET_RESET_FAILURE" @@ -17,14 +18,15 @@ export const errorCode = "DATASET_RESET_FAILURE" const datasetReset = async (req: Request, res: Response) => { const resmsgid = _.get(res, "resmsgid"); - const requestBody = req.body; + const category = req.body?.request?.category; + const datasetId = _.get(req, "params.datasetId") const msgid = _.get(req, ["body", "params", "msgid"]); try { const isRequestValid: Record = schemaValidation(req.body, DatasetResetRequestSchema) if (!isRequestValid.isValid) { const code = "DATASET_RESET_INPUT_INVALID" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) + logger.error({ code, apiId, msgid, category, resmsgid, message: isRequestValid.message }) return ResponseHandler.errorResponse({ code, message: isRequestValid.message, @@ -37,7 +39,7 @@ const datasetReset = async (req: Request, res: Response) => { if(_.isEmpty(dataset)) { const code = "DATASET_RESET_NO_DATASET" const message = `There are no live dataset exists with given dataset_id: ${datasetId}` - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: message }) + logger.error({ code, apiId, msgid, category, resmsgid, message: message }) return ResponseHandler.errorResponse({ code, message, @@ -45,11 +47,32 @@ const datasetReset = async (req: Request, res: Response) => { errCode: "BAD_REQUEST" } as ErrorObject, req, res); } - logger.debug(apiId, msgid, resmsgid, "dataset", dataset) + logger.debug(apiId, msgid, resmsgid, "dataset", dataset, category) + const isMasterDataset = _.get(dataset, "[0].type") == DatasetType.MasterDataset; + if(category == "processing") { + const pipeLineStatus = await getFlinkHealthStatus() + logger.debug({pipeLineStatus}) + if(pipeLineStatus == HealthStatus.UnHealthy){ + logger.debug("Restarting the pipeline") + await restartPipeline({"dataset": {"dataset_id": datasetId}}) + } + } else if(category == "query" && !isMasterDataset){ + const datasources = await getDataSources(datasetId) + const unHealthySupervisors = await getDruidIndexers(datasources, HealthStatus.UnHealthy) + const unHealthyDataSources = _.filter(unHealthySupervisors, (supervisor: any) => supervisor?.state == "SUSPENDED") + if(!_.isEmpty(unHealthyDataSources)){ + await restartDruidIndexers(unHealthyDataSources) + } + } + return ResponseHandler.successResponse(req, res, { + status: 200, data: { + "status": "Completed" + } + }); } catch (error: any) { - logger.error({ ...error, apiId, code: errorCode, msgid, requestBody, resmsgid }); + logger.error({ error, apiId, code: errorCode, msgid, category, resmsgid }); let errorMessage = error; const statusCode = _.get(error, "statusCode") if (!statusCode || statusCode == 500) { @@ -65,7 +88,7 @@ const getLiveDatasets = async (ids: string): Promise> => { return Dataset.findAll({ attributes: ['dataset_id', 'status', 'type'], where: { dataset_id: ids, status: DatasetStatus.Live }, raw: true }); } -const getDataSources = async (ids: Record): Promise> => { +const getDataSources = async (ids: string): Promise> => { return Datasource.findAll({ attributes: ['dataset_id', 'datasource'], where: { dataset_id: ids }, raw: true }); } diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 317b7ae4..c7e05559 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -217,7 +217,7 @@ const deleteSupervisors = async (configs: Record) => { } //RESTART_PIPELINE -const restartPipeline = async (config: Record) => { +export const restartPipeline = async (config: Record) => { const dataset_id = _.get(config, ["dataset", "dataset_id"]) return executeCommand(dataset_id, "RESTART_PIPELINE") } diff --git a/api-service/src/v2/services/HealthService.ts b/api-service/src/v2/services/HealthService.ts index 0f9244ca..1e659de7 100644 --- a/api-service/src/v2/services/HealthService.ts +++ b/api-service/src/v2/services/HealthService.ts @@ -57,7 +57,7 @@ const queryMetrics = (params: Record | string) => { export const getInfraHealth = async (isMasterDataset: boolean): Promise<{ components: any, status: string }> => { const postgres = await getPostgresStatus() const druid = await getDruidHealthStatus() - const flink = await getFlinkHealthStaus() + const flink = await getFlinkHealthStatus() const kafka = await getKafkaHealthStatus() let redis = HealthStatus.Healthy const components = [ @@ -77,7 +77,7 @@ export const getInfraHealth = async (isMasterDataset: boolean): Promise<{ compon export const getProcessingHealth = async (dataset: any): Promise<{ components: any, status: string }> => { const dataset_id = _.get(dataset, "dataset_id") const isMasterDataset = _.get(dataset, "type") == DatasetType.MasterDataset; - const flink = await getFlinkHealthStaus() + const flink = await getFlinkHealthStatus() const { count, health } = await getEventsProcessedToday(dataset_id, isMasterDataset) const processingDefaultThreshold = await SystemConfig.getThresholds("processing") // eslint-disable-next-line prefer-const @@ -252,8 +252,6 @@ const getDruidIndexerStatus = async (datasources: any,) => { logger.error(error) return { value: [], status: HealthStatus.UnHealthy } } - - } const getDruidDataourceStatus = async (datasourceId: string) => { @@ -287,7 +285,7 @@ const getKafkaHealthStatus = async () => { } -const getFlinkHealthStaus = async () => { +export const getFlinkHealthStatus = async () => { try { const responses = await Promise.all( [axios.get(config?.flink_job_configs?.masterdata_processor_job_manager_url as string + "/jobs"), @@ -517,5 +515,36 @@ const getQueriesFailedCount = async (datasetId: string) => { } +export const getDruidIndexers = async (datasources: any, status = HealthStatus.Healthy) => { + const results = await Promise.all(_.map(datasources, (datasource) => getDruidDataourceStatus(datasource["datasource"]))) + const indexers: any = [] + _.forEach(results, (result: any) => { + logger.debug({ result }) + const sourceStatus = _.get(result, "payload.state") == "RUNNING" ? HealthStatus.Healthy : HealthStatus.UnHealthy + logger.debug({ sourceStatus }) + if (sourceStatus == status) { + indexers.push( + { + "type": "druid", + "datasource": _.get(result, "id"), + "status": sourceStatus, + "state": _.get(result, "payload.state") + } + ) + } + }) + return indexers +} + +const restartDruidSupervisors = async (datasourceId: string) => { + logger.debug(datasourceId) + const { data } = await druidHttpService.post(`/druid/indexer/v1/supervisor/${datasourceId}/resume`) + return data; +} +export const restartDruidIndexers = async (datasources: any) => { + logger.debug({datasources}) + const results = await Promise.all(_.map(datasources, (datasource) => restartDruidSupervisors(datasource["datasource"]))) + logger.debug({restartIndexerResponse: results}) +} init().catch(err => logger.error(err)) \ No newline at end of file From 99d29e6a2ac574a31aa12766570dc09b591c7a52 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 26 Jun 2024 13:10:41 +0530 Subject: [PATCH 004/235] Issue #OBS-I2 fix: linting --- api-service/src/v2/controllers/DatasetReset/DatasetReset.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts b/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts index 7462b6f7..29376ab2 100644 --- a/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts @@ -85,11 +85,11 @@ const datasetReset = async (req: Request, res: Response) => { } const getLiveDatasets = async (ids: string): Promise> => { - return Dataset.findAll({ attributes: ['dataset_id', 'status', 'type'], where: { dataset_id: ids, status: DatasetStatus.Live }, raw: true }); + return Dataset.findAll({ attributes: ["dataset_id", "status", "type"], where: { dataset_id: ids, status: DatasetStatus.Live }, raw: true }); } const getDataSources = async (ids: string): Promise> => { - return Datasource.findAll({ attributes: ['dataset_id', 'datasource'], where: { dataset_id: ids }, raw: true }); + return Datasource.findAll({ attributes: ["dataset_id", "datasource"], where: { dataset_id: ids }, raw: true }); } export default datasetReset; \ No newline at end of file From 715e5a56f618d25784eb57f4e712d5fab76f1648 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 4 Jul 2024 11:23:44 +0530 Subject: [PATCH 005/235] feat 000: Dataset create API refactoring --- api-service/src/app.ts | 2 + api-service/src/v1/helpers/ResponseHandler.ts | 5 + api-service/src/v1/routes/Router.ts | 8 +- api-service/src/v2/configs/Config.ts | 4 + .../src/v2/configs/DatasetConfigDefault.ts | 71 ++--- .../src/v2/connections/databaseConnection.ts | 4 +- .../DatasetCreate/DatasetCreate.ts | 155 ++++------- .../DatasetCreateValidationSchema.json | 139 ++++++++-- .../v2/controllers/DatasetRead/DatasetRead.ts | 10 +- .../src/v2/metrics/prometheus/helpers.ts | 9 + api-service/src/v2/middlewares/errors.ts | 11 + .../src/v2/models/ConnectorInstances.ts | 54 ++++ .../src/v2/models/ConnectorInstancesDraft.ts | 58 ++++ .../src/v2/models/ConnectorRegistry.ts | 86 ++++++ api-service/src/v2/models/Dataset.ts | 17 +- api-service/src/v2/models/DatasetDraft.ts | 12 +- .../src/v2/models/DatasetSourceConfigDraft.ts | 3 - api-service/src/v2/models/DatasourceDraft.ts | 3 - .../src/v2/models/TransformationDraft.ts | 4 - api-service/src/v2/services/CipherService.ts | 30 ++ api-service/src/v2/services/DatasetService.ts | 257 +++++++++++------- 21 files changed, 638 insertions(+), 304 deletions(-) create mode 100644 api-service/src/v2/middlewares/errors.ts create mode 100644 api-service/src/v2/models/ConnectorInstances.ts create mode 100644 api-service/src/v2/models/ConnectorInstancesDraft.ts create mode 100644 api-service/src/v2/models/ConnectorRegistry.ts create mode 100644 api-service/src/v2/services/CipherService.ts diff --git a/api-service/src/app.ts b/api-service/src/app.ts index bc7aad96..7783a4aa 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -10,6 +10,7 @@ import { interceptAuditEvents } from "./v1/services/telemetry"; import { queryService } from "./v1/routes/Router"; import { routesConfig } from "./v1/configs/RoutesConfig"; import { QueryValidator } from "./v1/validators/QueryValidator"; +import { errorHandler } from "./v2/middlewares/errors"; const app: Application = express(); const queryValidator = new QueryValidator(); @@ -23,6 +24,7 @@ const services = { app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); +app.use(errorHandler) app.set("queryServices", services); loadExtensions(app) diff --git a/api-service/src/v1/helpers/ResponseHandler.ts b/api-service/src/v1/helpers/ResponseHandler.ts index e2d35471..89b21880 100644 --- a/api-service/src/v1/helpers/ResponseHandler.ts +++ b/api-service/src/v1/helpers/ResponseHandler.ts @@ -43,6 +43,11 @@ const ResponseHandler = { entity && onSuccess(req, res) res.status(result.status).send(result.data); }, + + goneResponse: (req: Request, res: Response) => { + const { id, entity } = req as any; + res.status(httpStatus.GONE).json({ id: id, ver: "v1", ts: Date.now(), params: { status: constants.STATUS.FAILURE, errmsg: "v1 APIs have been replace by /v2 APIs. Please refer to this link for more information" }, responseCode: httpStatus["410_NAME"] }) + } } export { ResponseHandler }; diff --git a/api-service/src/v1/routes/Router.ts b/api-service/src/v1/routes/Router.ts index 2da73a9c..d716977a 100644 --- a/api-service/src/v1/routes/Router.ts +++ b/api-service/src/v1/routes/Router.ts @@ -35,8 +35,14 @@ export const globalCache: any = new Map() export const healthService = new HealthService(dbConnector, kafkaConnector, httpDruidConnector) export const router = express.Router() dbConnector.init() -/** Query API(s) */ +router.all("/datasets/v1/*", ResponseHandler.goneResponse) +router.all("/dataset/v1/*", ResponseHandler.goneResponse) +router.all("/datasources/v1/*", ResponseHandler.goneResponse) +router.all("/data/v1/*", ResponseHandler.goneResponse) +router.all("/template/v1/*", ResponseHandler.goneResponse) + +/** Query API(s) */ router.post([`${routesConfig.query.native_query.path}`, `${routesConfig.query.native_query_with_params.path}`,], ResponseHandler.setApiId(routesConfig.query.native_query.api_id), telemetryAuditStart({ action: telemetryActions.nativeQuery, operationType: OperationType.GET }), onRequest({ entity: Entity.Data_out }), validationService.validateRequestBody, validationService.validateQuery, queryService.executeNativeQuery); router.post([`${routesConfig.query.sql_query.path}`, `${routesConfig.query.sql_query_with_params.path}`,], ResponseHandler.setApiId(routesConfig.query.sql_query.api_id), telemetryAuditStart({ action: telemetryActions.sqlQuery, operationType: OperationType.GET }), onRequest({ entity: Entity.Data_out }), validationService.validateRequestBody, validationService.validateQuery, queryService.executeSqlQuery); diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index 447ee036..0073ae95 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -94,5 +94,9 @@ export const config = { "host": process.env.command_service_host || "http://localhost", "port": parseInt(process.env.command_service_port || "8000"), "path": process.env.command_service_path || "/system/v1/dataset/command" + }, + "encryption_config": { + "encryption_key": process.env.encryption_key || "strong_encryption_key_to_encrypt", + "encryption_algorithm": process.env.encryption_algorithm || "aes-256-ecb", } } diff --git a/api-service/src/v2/configs/DatasetConfigDefault.ts b/api-service/src/v2/configs/DatasetConfigDefault.ts index 850d0a05..768a7ffd 100644 --- a/api-service/src/v2/configs/DatasetConfigDefault.ts +++ b/api-service/src/v2/configs/DatasetConfigDefault.ts @@ -2,58 +2,14 @@ import { config } from "./Config"; import { DatasetStatus, ValidationMode } from "../types/DatasetModels"; import { ingestionConfig } from "./IngestionConfig"; -export const defaultMasterConfig = { - "validation_config": { - "validate": true, - "mode": ValidationMode.Strict, - }, - "extraction_config": { - "is_batch_event": false, - "extraction_key": "", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - }, - "denorm_config": { - "redis_db_host": config.redis_config.denorm_redis_host, - "redis_db_port": config.redis_config.denorm_redis_port, - "denorm_fields": [] - }, - "router_config": { - "topic": "" - }, - "tags": [], - "dataset_config": { - "data_key": "", - "timestamp_key": ingestionConfig.indexCol["Event Arrival Time"], - "entry_topic": config.telemetry_service_config.kafka.topics.createMasterDataset, - "redis_db_host": config.redis_config.denorm_redis_host, - "redis_db_port": config.redis_config.denorm_redis_port, - "index_data": true, - "redis_db": 3, - "file_upload_path": [] - }, - "status": DatasetStatus.Draft, - "version": 1, - "created_by": "SYSTEM", - "updated_by": "SYSTEM" -} - export const defaultDatasetConfig = { "validation_config": { "validate": true, "mode": ValidationMode.Strict, }, "extraction_config": { - "is_batch_event": false, - "extraction_key": "", + "is_batch_event": true, + "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", @@ -75,15 +31,24 @@ export const defaultDatasetConfig = { }, "tags": [], "dataset_config": { - "data_key": "", - "timestamp_key": ingestionConfig.indexCol["Event Arrival Time"], - "entry_topic": config.telemetry_service_config.kafka.topics.createDataset, - "redis_db_host": config.redis_config.dedup_redis_host, - "redis_db_port": config.redis_config.dedup_redis_port, - "index_data": true, - "redis_db": 0, + "indexing_config": { + "olap_store_enabled": true, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "data_key": "", + "partition_key": "", + "timestamp_key": ingestionConfig.indexCol["Event Arrival Time"] + }, + "cache_config": { + "redis_db_host": config.redis_config.dedup_redis_host, + "redis_db_port": config.redis_config.dedup_redis_port, + "redis_db": 0 + }, "file_upload_path": [] }, + "entry_topic": config.telemetry_service_config.kafka.topics.createDataset, "status": DatasetStatus.Draft, "api_version": "v2", "version": 1, diff --git a/api-service/src/v2/connections/databaseConnection.ts b/api-service/src/v2/connections/databaseConnection.ts index a32dd9fa..17df7e1b 100644 --- a/api-service/src/v2/connections/databaseConnection.ts +++ b/api-service/src/v2/connections/databaseConnection.ts @@ -5,8 +5,8 @@ const { database, host, password, port, username } = connectionConfig.postgres export const sequelize = new Sequelize({ database, password, username: username, dialect: "postgres", host, port: +port, pool: { - max: 5, - min: 0, + max: 2, + min: 1, acquire: 30000, idle: 10000 } diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index e2bb2a53..d75a03c4 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -1,126 +1,65 @@ import { Request, Response } from "express"; +import httpStatus from "http-status"; import logger from "../../logger"; -import { generateDataSource, getDefaultValue, getDraftDataset, getDuplicateConfigs, getDuplicateDenormKey, setReqDatasetId } from "../../services/DatasetService"; -import _ from "lodash"; +import { datasetService } from "../../services/DatasetService"; import DatasetCreate from "./DatasetCreateValidationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; -import { DatasetDraft } from "../../models/DatasetDraft"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import httpStatus from "http-status"; import { ErrorObject } from "../../types/ResponseModel"; -import { DatasourceDraft } from "../../models/DatasourceDraft"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; export const apiId = "api.datasets.create" -const datasetCreate = async (req: Request, res: Response) => { - const requestBody = req.body - const msgid = _.get(req, ["body", "params", "msgid"]); - const resmsgid = _.get(res, "resmsgid"); - try { - const datasetId = _.get(req, ["body", "request", "dataset_id"]) - setReqDatasetId(req, datasetId) - - const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) - - if (!isRequestValid.isValid) { - const code = "DATASET_INVALID_INPUT" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetBody = req.body.request; - const isDataSetExists = await checkDatasetExists(_.get(datasetBody, ["dataset_id"])); - if (isDataSetExists) { - const code = "DATASET_EXISTS" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Dataset Already exists with id:${_.get(datasetBody, "dataset_id")}` }) - return ResponseHandler.errorResponse({ - code, - message: "Dataset already exists", - statusCode: 409, - errCode: "CONFLICT" - } as ErrorObject, req, res); - } - - const duplicateDenormKeys = getDuplicateDenormKey(_.get(datasetBody, "denorm_config")) - if (!_.isEmpty(duplicateDenormKeys)) { - const code = "DATASET_DUPLICATE_DENORM_KEY" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Duplicate denorm output fields found. Duplicate Denorm out fields are [${duplicateDenormKeys}]` }) - return ResponseHandler.errorResponse({ - code, - statusCode: 400, - message: "Duplicate denorm key found", - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetPayload: any = await getDefaultValue(datasetBody); - const data = { ...datasetPayload, version_key: Date.now().toString() } - - const response = await DatasetDraft.create(data) - - const { dataset_config, denorm_config, transformation_config, data_schema, id, dataset_id } = data - const datasourcePayload = await generateDataSource({ indexCol: _.get(dataset_config, ["timestamp_key"]), data_schema, id, dataset_id, denorm_config, transformation_config, action:"create" }) - await DatasourceDraft.create(datasourcePayload) - logger.info({ apiId, message: `Datasource created successsfully for the dataset:${id}` }) - - const transformationConfig: any = getTransformationConfig({ transformationPayload: _.get(datasetBody, "transformations_config"), datasetId: _.get(datasetPayload, "id") }) - if (!_.isEmpty(transformationConfig)) { - await DatasetTransformationsDraft.bulkCreate(transformationConfig); - logger.info({ apiId, message: `Dataset transformations records created successsfully for dataset:${id}` }) - } - - const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: data.version_key } - logger.info({ apiId, msgid, requestBody, resmsgid, message: `Dataset Created Successfully with id:${_.get(response, ["dataValues", "id"])}`, response: responseData }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); - } catch (error: any) { - const code = _.get(error, "code") || "DATASET_CREATION_FAILURE" - logger.error({ ...error, apiId, code, msgid, requestBody, resmsgid }) - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code, message: "Failed to create dataset" } - } - ResponseHandler.errorResponse(errorMessage, req, res); +const isValidRequest = async (req: Request, res: Response): Promise => { + + const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) + if (!isRequestValid.isValid) { + logger.error({ code: "DATASET_INVALID_INPUT", apiId, body: req.body, message: isRequestValid.message }) + ResponseHandler.errorResponse({ + code: "DATASET_INVALID_INPUT", + message: isRequestValid.message, + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + return false; } -} - -const checkDatasetExists = async (dataset_id: string): Promise => { - const datasetExists = await getDraftDataset(dataset_id) - if (datasetExists) { - return true; - } else { - return false + const datasetId = _.get(req, ["body", "request", "dataset_id"]) + const isDataSetExists = await datasetService.checkDatasetExists(datasetId); + if (isDataSetExists) { + logger.error({ code: "DATASET_EXISTS", apiId, body: req.body, message: `Dataset Already exists with id:${datasetId}` }) + ResponseHandler.errorResponse({ + code: "DATASET_EXISTS", + message: "Dataset already exists", + statusCode: 409, + errCode: "CONFLICT" + } as ErrorObject, req, res); + return false; } -} - -const getTransformationConfig = (configs: Record): Record => { - const { transformationPayload, datasetId } = configs - if (transformationPayload) { - let transformations: any = [] - const transformationFieldKeys = _.flatten(_.map(transformationPayload, fields => _.get(fields, ["field_key"]))) - const duplicateFieldKeys: Array = getDuplicateConfigs(transformationFieldKeys) + const duplicateDenormKeys = datasetService.getDuplicateDenormKey(_.get(req, ["body", "request", "denorm_config"])) + if (!_.isEmpty(duplicateDenormKeys)) { + const code = "DATASET_DUPLICATE_DENORM_KEY" + logger.error({ code: "DATASET_DUPLICATE_DENORM_KEY", apiId, body: req.body, message: `Duplicate denorm output fields found. Duplicate Denorm out fields are [${duplicateDenormKeys}]` }) + ResponseHandler.errorResponse({ + code: "DATASET_DUPLICATE_DENORM_KEY", + statusCode: 400, + message: "Duplicate denorm key found", + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + return false; + } - if (!_.isEmpty(duplicateFieldKeys)) { - logger.info({ apiId, message: `Duplicate transformations provided by user are [${duplicateFieldKeys}]` }) - } + return true; +} - _.forEach(transformationPayload, payload => { - const fieldKey = _.get(payload, "field_key") - const transformationExists = _.some(transformations, field => _.get(field, "field_key") == fieldKey) - if (!transformationExists) { - transformations = _.flatten(_.concat(transformations, { ...payload, id: `${datasetId}_${fieldKey}`, dataset_id: datasetId })) - } - }) - return transformations +const datasetCreate = async (req: Request, res: Response) => { + + const isRequestValid = await isValidRequest(req, res) + if(!isRequestValid) { + return; } - return [] + + const dataset = await datasetService.create(req.body.request); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } export default datasetCreate; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index 07582f41..f12eb65d 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -30,7 +30,7 @@ }, "type": { "type": "string", - "enum": ["dataset", "master-dataset"] + "enum": ["event", "transaction", "master"] }, "name": { "type": "string", @@ -40,11 +40,13 @@ "type": "object", "properties": { "validate": { - "type": "boolean" + "type": "boolean", + "default": true }, "mode": { "type": "string", - "enum": ["Strict", "IgnoreNewFields"] + "enum": ["Strict", "IgnoreNewFields"], + "default": "Strict" } }, "required": ["validate", "mode"], @@ -58,13 +60,15 @@ }, "extraction_key": { "type": "string", + "default": "events", "minLength": 1 }, "dedup_config": { "type": "object", "properties": { "drop_duplicates": { - "type": "boolean" + "type": "boolean", + "default": false }, "dedup_key": { "type": "string", @@ -82,7 +86,8 @@ "type": "object", "properties": { "drop_duplicates": { - "type": "boolean" + "type": "boolean", + "default": true }, "dedup_key": { "type": "string", @@ -123,6 +128,10 @@ "items": { "type": "object", "properties": { + "denorm_dataset": { + "type": "string", + "minLength": 1 + }, "denorm_key": { "type": "string", "minLength": 1 @@ -130,9 +139,20 @@ "denorm_out_field": { "type": "string", "minLength": 1 + }, + "jsonata_expr": { + "type": "string", + "minLength": 1 } }, - "required": ["denorm_key", "denorm_out_field"], + "oneOf": [ + { + "required": ["denorm_dataset", "denorm_out_field", "denorm_key"] + }, + { + "required": ["denorm_dataset", "denorm_out_field", "jsonata_expr"] + } + ], "additionalProperties": false } } @@ -143,22 +163,55 @@ "dataset_config": { "type": "object", "properties": { - "data_key": { - "type": "string" - }, - "timestamp_key": { - "type": "string", - "minLength": 1 - }, "file_upload_path": { "type": "array", "items": { "type": "string", "minLength": 1 } + }, + "dataset_tz": { + "type": "string" + }, + "indexing_config": { + "type": "object", + "properties": { + "olap_store_enabled": { + "type": "boolean", + "default": false + }, + "lakehouse_enabled": { + "type": "boolean", + "default": true + }, + "cache_enabled": { + "type": "boolean", + "default": false + } + }, + "additionalProperties": false + }, + "keys_config": { + "type": "object", + "properties": { + "data_key": { + "type": "string" + }, + "partition_key": { + "type": "string" + }, + "timestamp_key": { + "type": "string" + }, + "timestamp_format": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false } }, - "required": ["data_key", "timestamp_key"], + "required": ["indexing_config", "keys_config"], "additionalProperties": false }, "transformations_config": { @@ -182,17 +235,30 @@ "minLength": 1 }, "condition": { - "type": "string" + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "expr"], + "additionalProperties": false }, - "dataType": { + "datatype": { "type": "string" + }, + "category": { + "type": "string", + "enum": ["pii", "transform", "derived"] } }, - "additionalProperties": false, - "required": [ - "type", - "expr" - ] + "required": ["type", "expr"], + "additionalProperties": false }, "mode": { "type": "string", @@ -200,29 +266,48 @@ "Strict", "Lenient" ] - }, - "metadata": { - "type": "object" } }, "additionalProperties": false, "required": [ "field_key", "transformation_function", - "mode", - "metadata" + "mode" ] } }, + "connectors_config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "connector_id": { + "type": "string", + "minLength": 1 + }, + "connector_config": { + "type": "object" + }, + "operations_config": { + "type": "object" + } + }, + "additionalProperties": false, + "required": ["connector_id", "connector_config"] + } + }, "tags": { "type": "array", "items": { "type": "string", "minLength":1 } + }, + "sample_data": { + "type": "object" } }, - "required": ["dataset_id", "type", "name", "data_schema"], + "required": ["dataset_id", "name"], "additionalProperties": false } }, diff --git a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts index 858df16f..11b4ef01 100644 --- a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts @@ -10,7 +10,7 @@ import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; import { DatasetTransformations } from "../../models/Transformation"; import { DatasetDraft } from "../../models/DatasetDraft"; import { Dataset } from "../../models/Dataset"; -import { getDataset, setReqDatasetId } from "../../services/DatasetService"; +import { datasetService } from "../../services/DatasetService"; import { Datasource } from "../../models/Datasource"; import { DatasetSourceConfig } from "../../models/DatasetSourceConfig"; import { DatasourceDraft } from "../../models/DatasourceDraft"; @@ -76,6 +76,12 @@ const datasetRead = async (req: Request, res: Response) => { } } +const setReqDatasetId = (req: Request, dataset_id: string) => { + if (dataset_id) { + return _.set(req, "dataset_id", dataset_id) + } +} + const getDatasetModel = (status: string | any) => { if (status === DatasetStatus.Draft || status === DatasetStatus.ReadyToPublish) { return DatasetDraft; @@ -126,7 +132,7 @@ const getTransfomationModel = (status: string) => { const getDatasetByLive = async (dataset_id: string) => { const draftRecord = await DatasetDraft.findOne({ where: { id: dataset_id }, raw: true }) - const liveDataset = await getDataset(dataset_id, true) + const liveDataset = await datasetService.getDataset(dataset_id, true) if (_.isEmpty(liveDataset)) { throw { code: "DATASET_NOT_FOUND", diff --git a/api-service/src/v2/metrics/prometheus/helpers.ts b/api-service/src/v2/metrics/prometheus/helpers.ts index 0a443f45..5e0442c1 100644 --- a/api-service/src/v2/metrics/prometheus/helpers.ts +++ b/api-service/src/v2/metrics/prometheus/helpers.ts @@ -33,6 +33,15 @@ export const onFailure = (req: any, res: Response) => { incrementFailedApiCalls({ labels }); } +export const onGone = (req: any, res: Response) => { + const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) + const { statusCode = 410 } = res + const labels = { ...metricLabels, status: statusCode } + duration && setQueryResponseTime({ duration, labels }); + incrementApiCalls({ labels }) + incrementFailedApiCalls({ labels }); +} + const getMetricLabels = (req: any, res: Response) => { const { id, entity, url, startTime } = req; const { statusCode = 200 } = res diff --git a/api-service/src/v2/middlewares/errors.ts b/api-service/src/v2/middlewares/errors.ts new file mode 100644 index 00000000..e52f05f3 --- /dev/null +++ b/api-service/src/v2/middlewares/errors.ts @@ -0,0 +1,11 @@ +import { NextFunction, Request, Response } from "express"; +import logger from "../logger"; +import { ResponseHandler } from "../helpers/ResponseHandler"; +import _ from "lodash"; + +export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => { + + logger.error({ path: req.url, req: req.body , ...err }) + let errorMessage = {name: err.name, message: err.message}; + ResponseHandler.errorResponse(errorMessage, req, res); +}; \ No newline at end of file diff --git a/api-service/src/v2/models/ConnectorInstances.ts b/api-service/src/v2/models/ConnectorInstances.ts new file mode 100644 index 00000000..df840042 --- /dev/null +++ b/api-service/src/v2/models/ConnectorInstances.ts @@ -0,0 +1,54 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; + +export const ConnectorInstances = sequelize.define("connector_instances", { + id: { + type: DataTypes.STRING, + primaryKey: true + }, + dataset_id: { + type: DataTypes.STRING + }, + connector_id: { + type: DataTypes.STRING + }, + data_format: { + type: DataTypes.STRING, + defaultValue: "json" + }, + connector_config: { + type: DataTypes.STRING + }, + operations_config: { + type: DataTypes.JSON, + defaultValue: {} + }, + status: { + type: DataTypes.ENUM("Publishing", "Live", "Retired") + }, + connector_state: { + type: DataTypes.JSON, + defaultValue: {} + }, + connector_stats: { + type: DataTypes.JSON, + defaultValue: {} + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + published_date: { + type: DataTypes.NUMBER + } +}, { + tableName: "connector_instances", + timestamps: true, + createdAt: "created_date", + updatedAt: "updated_date", + paranoid: true +}) \ No newline at end of file diff --git a/api-service/src/v2/models/ConnectorInstancesDraft.ts b/api-service/src/v2/models/ConnectorInstancesDraft.ts new file mode 100644 index 00000000..7d7783c5 --- /dev/null +++ b/api-service/src/v2/models/ConnectorInstancesDraft.ts @@ -0,0 +1,58 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; + +export const ConnectorInstancesDraft = sequelize.define("connector_instances_draft", { + id: { + type: DataTypes.STRING, + primaryKey: true, + allowNull: false, + autoIncrement: true + }, + dataset_id: { + type: DataTypes.STRING, + allowNull: false + }, + connector_id: { + type: DataTypes.STRING, + allowNull: false + }, + data_format: { + type: DataTypes.STRING, + defaultValue: "json", + allowNull: false + }, + connector_config: { + type: DataTypes.STRING, + allowNull: false + }, + operations_config: { + type: DataTypes.JSON, + defaultValue: {}, + allowNull: false + }, + status: { + type: DataTypes.ENUM("Draft", "Publishing", "Live", "Retired"), + defaultValue: "Draft", + allowNull: false + }, + validation_config: { + type: DataTypes.JSON, + defaultValue: {} + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + published_date: { + type: DataTypes.NUMBER + } +}, { + tableName: "connector_instances_draft", + timestamps: true, + createdAt: "created_date", + updatedAt: "updated_date" +}) \ No newline at end of file diff --git a/api-service/src/v2/models/ConnectorRegistry.ts b/api-service/src/v2/models/ConnectorRegistry.ts new file mode 100644 index 00000000..64ecad46 --- /dev/null +++ b/api-service/src/v2/models/ConnectorRegistry.ts @@ -0,0 +1,86 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; + +export const ConnectorRegistry = sequelize.define("connector_registry", { + id: { + type: DataTypes.STRING, + primaryKey: true + }, + connector_id: { + type: DataTypes.STRING, + allowNull: false, + unique: "connector_registry_unique" + }, + name: { + type: DataTypes.STRING, + allowNull: false + }, + type: { + type: DataTypes.STRING, + allowNull: false + }, + category: { + type: DataTypes.STRING, + allowNull: false + }, + version: { + type: DataTypes.STRING, + allowNull: false, + unique: "connector_registry_unique" + }, + description: { + type: DataTypes.STRING + }, + technology: { + type: DataTypes.STRING, + allowNull: false + }, + runtime: { + type: DataTypes.STRING, + allowNull: false + }, + licence: { + type: DataTypes.STRING, + allowNull: false + }, + owner: { + type: DataTypes.STRING, + allowNull: false + }, + iconURL: { + type: DataTypes.STRING + }, + status: { + type: DataTypes.ENUM("Draft", "InValidation", "Live", "Retired"), + defaultValue: "Draft", + }, + ui_spec: { + type: DataTypes.JSON, + defaultValue: {} + }, + source_url: { + type: DataTypes.STRING, + allowNull: false + }, + source: { + type: DataTypes.STRING, + allowNull: false + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM" + }, + live_date: { + type: DataTypes.DATE + } +}, { + tableName: "connector_registry", + timestamps: true, + createdAt: "created_date", + updatedAt: "updated_date", + paranoid: true +}) \ No newline at end of file diff --git a/api-service/src/v2/models/Dataset.ts b/api-service/src/v2/models/Dataset.ts index aebcc0fb..591bdc01 100644 --- a/api-service/src/v2/models/Dataset.ts +++ b/api-service/src/v2/models/Dataset.ts @@ -48,7 +48,7 @@ export const Dataset = sequelize.define("datasets", { defaultValue: {} }, status: { - type: DataTypes.ENUM("Draft", "Live", "Retired", "Publish"), + type: DataTypes.ENUM("Draft", "Publish", "Live", "Retired", "Archiving", "Archived"), defaultValue: "Draft", }, created_by: { @@ -61,10 +61,25 @@ export const Dataset = sequelize.define("datasets", { }, data_version: { type: DataTypes.NUMBER + }, + api_version: { + type: DataTypes.STRING, + }, + version: { + type: DataTypes.NUMBER + }, + fields_set: { + type: DataTypes.JSON, + defaultValue: {} + }, + sample_data: { + type: DataTypes.JSON, + defaultValue: {} } }, { tableName: "datasets", timestamps: true, createdAt: "created_date", updatedAt: "updated_date", + paranoid: true }) \ No newline at end of file diff --git a/api-service/src/v2/models/DatasetDraft.ts b/api-service/src/v2/models/DatasetDraft.ts index a77a4259..9892032f 100644 --- a/api-service/src/v2/models/DatasetDraft.ts +++ b/api-service/src/v2/models/DatasetDraft.ts @@ -80,10 +80,6 @@ export const DatasetDraft = sequelize.define("datasets_draft", { allowNull: false, defaultValue: "SYSTEM", }, - published_date: { - type: DataTypes.STRING, - allowNull: true - }, client_state: { type: DataTypes.JSON, defaultValue: {} @@ -95,6 +91,14 @@ export const DatasetDraft = sequelize.define("datasets_draft", { api_version: { type: DataTypes.STRING, defaultValue: "v2" + }, + fields_set: { + type: DataTypes.JSON, + defaultValue: {} + }, + sample_data: { + type: DataTypes.JSON, + defaultValue: {} } }, { timestamps: true, diff --git a/api-service/src/v2/models/DatasetSourceConfigDraft.ts b/api-service/src/v2/models/DatasetSourceConfigDraft.ts index 8807b059..df84c1a6 100644 --- a/api-service/src/v2/models/DatasetSourceConfigDraft.ts +++ b/api-service/src/v2/models/DatasetSourceConfigDraft.ts @@ -32,9 +32,6 @@ export const DatasetSourceConfigDraft = sequelize.define("dataset_source_config_ updated_by: { type: DataTypes.STRING, defaultValue: "SYSTEM", - }, - published_date: { - type: DataTypes.TIME } }, { tableName: "dataset_source_config_draft", diff --git a/api-service/src/v2/models/DatasourceDraft.ts b/api-service/src/v2/models/DatasourceDraft.ts index b14123e2..7b211358 100644 --- a/api-service/src/v2/models/DatasourceDraft.ts +++ b/api-service/src/v2/models/DatasourceDraft.ts @@ -52,9 +52,6 @@ export const DatasourceDraft = sequelize.define("datasources_draft", { type: DataTypes.STRING, defaultValue: "SYSTEM", }, - published_date: { - type: DataTypes.TIME - }, metadata: { type: DataTypes.JSON, defaultValue: { "aggregated": false, "granularity": "day" } diff --git a/api-service/src/v2/models/TransformationDraft.ts b/api-service/src/v2/models/TransformationDraft.ts index 601d5c59..55cb0264 100644 --- a/api-service/src/v2/models/TransformationDraft.ts +++ b/api-service/src/v2/models/TransformationDraft.ts @@ -41,10 +41,6 @@ export const DatasetTransformationsDraft = sequelize.define("dataset_transformat type: DataTypes.TEXT, allowNull: false, defaultValue: "SYSTEM", - }, - published_date: { - type: DataTypes.DATE, - defaultValue: null } }, { timestamps: true, diff --git a/api-service/src/v2/services/CipherService.ts b/api-service/src/v2/services/CipherService.ts new file mode 100644 index 00000000..b0b86c25 --- /dev/null +++ b/api-service/src/v2/services/CipherService.ts @@ -0,0 +1,30 @@ +import crypto from 'crypto'; +import { config } from '../configs/Config'; + +class CipherService { + public encrypt(data: string) { + const cipher = crypto.createCipheriv( + config.encryption_config.encryption_algorithm, + config.encryption_config.encryption_key, + "", + ) + const toEncrypt = Buffer.from(data, 'utf8'); + let encryptedString = cipher.update(toEncrypt); + encryptedString = Buffer.concat([encryptedString, cipher.final()]) + return encryptedString.toString('base64'); + } + + public decrypt(data: string) { + const decipher = crypto.createDecipheriv( + config.encryption_config.encryption_algorithm, + config.encryption_config.encryption_key, + "", + ) + const encryptedText = Buffer.from(data, 'base64'); + let decryptedString = decipher.update(encryptedText); + decryptedString = Buffer.concat([decryptedString, decipher.final()]) + return decryptedString.toString(); + } +} + +export const cipherService = new CipherService() \ No newline at end of file diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 8a8b5c81..30f79f15 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -1,118 +1,183 @@ -import { Dataset } from "../models/Dataset"; import _ from "lodash"; +import { v4 } from "uuid"; +import logger from "../logger"; +import { cipherService } from "./CipherService"; +import { sequelize } from "../connections/databaseConnection"; +import { defaultDatasetConfig } from "../configs/DatasetConfigDefault"; +import { Dataset } from "../models/Dataset"; import { DatasetDraft } from "../models/DatasetDraft"; -import { DatasetTransformationsDraft } from "../models/TransformationDraft"; -import { Request } from "express"; -import { generateIngestionSpec } from "./IngestionService"; -import { ingestionConfig } from "../configs/IngestionConfig"; import { DatasetTransformations } from "../models/Transformation"; -import { getUpdatedSchema } from "./DatasourceService"; -import { DatasetType } from "../types/DatasetModels"; -import { defaultDatasetConfig, defaultMasterConfig } from "../configs/DatasetConfigDefault"; -import { query } from "../connections/databaseConnection"; - -export const getDataset = async (datasetId: string, raw = false): Promise => { - const dataset = await Dataset.findOne({ - where: { - id: datasetId, - }, - raw: raw - }); - return dataset -} +import { ConnectorInstancesDraft } from "../models/ConnectorInstancesDraft"; +import { DatasetTransformationsDraft } from "../models/TransformationDraft"; +import { DatasetSourceConfigDraft } from "../models/DatasetSourceConfigDraft"; -export const getDuplicateDenormKey = (denormConfig: Record): Array => { - if (denormConfig && _.isArray(_.get(denormConfig, "denorm_fields"))) { - const denormFields = _.get(denormConfig, "denorm_fields") - const denormOutKeys = _.map(denormFields, field => _.get(field, "denorm_out_field")) - const duplicateDenormKeys: Array = _.filter(denormOutKeys, (item: string, index: number) => _.indexOf(denormOutKeys, item) !== index); - return duplicateDenormKeys; +class DatasetService { + + getDataset = async (datasetId: string, raw = false): Promise => { + const dataset = await Dataset.findOne({ + where: { + id: datasetId, + }, + raw: raw + }); + return dataset } - return [] -} -export const getDraftDataset = async (dataset_id: string) => { - return DatasetDraft.findOne({ where: { dataset_id }, raw: true }); -} + getDuplicateDenormKey = (denormConfig: Record): Array => { + if (denormConfig && _.isArray(_.get(denormConfig, "denorm_fields"))) { + const denormFields = _.get(denormConfig, "denorm_fields") + const denormOutKeys = _.map(denormFields, field => _.get(field, "denorm_out_field")) + const duplicateDenormKeys: Array = _.filter(denormOutKeys, (item: string, index: number) => _.indexOf(denormOutKeys, item) !== index); + return duplicateDenormKeys; + } + return [] + } -export const getDraftTransformations = async (dataset_id: string) => { - return DatasetTransformationsDraft.findAll({ where: { dataset_id }, raw: true }); -} + checkDatasetExists = async (dataset_id: string): Promise => { + const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes:["id"], raw: true }); + if (draft === null) { + const live = await Dataset.findOne({ where: { id: dataset_id }, attributes:["id"], raw: true }); + return !(live === null) + } else { + return true; + } + } -export const getTransformations = async (dataset_id: string) => { - return DatasetTransformations.findAll({ where: { dataset_id }, raw: true }); -} + getDraftDataset = async (dataset_id: string) => { + return DatasetDraft.findOne({ where: { dataset_id }, raw: true }); + } -export const setReqDatasetId = (req: Request, dataset_id: string) => { - if (dataset_id) { - return _.set(req, "dataset_id", dataset_id) + getDraftTransformations = async (dataset_id: string) => { + return DatasetTransformationsDraft.findAll({ where: { dataset_id }, raw: true }); } -} -export const getDuplicateConfigs = (configs: Array) => { - return _.filter(configs, (item: string, index: number) => _.indexOf(configs, item) !== index); -} + getTransformations = async (dataset_id: string) => { + return DatasetTransformations.findAll({ where: { dataset_id }, raw: true }); + } -export const generateDataSource = async (payload: Record) => { - const { id } = payload - const updatedSchema = await getUpdatedSchema(payload) - const ingestionSpec = generateIngestionSpec({ ...payload, data_schema: updatedSchema }) - const dataSource = getDataSource({ ingestionSpec, id }) - return dataSource -} + create = async (datasetReq: Record): Promise> => { -const getDataSource = (ingestionPayload: Record) => { - const { ingestionSpec, id } = ingestionPayload - const dataSource = `${id}_${_.toLower(ingestionConfig.granularitySpec.segmentGranularity)}` - const dataSourceId = `${id}_${dataSource}` - return { - id: dataSourceId, - datasource: dataSource, - dataset_id: id, - ingestion_spec: ingestionSpec, - datasource_ref: dataSource + let transaction; + try { + transaction = await sequelize.transaction() + const dataset = await this.createDataset(datasetReq, transaction); + await this.createDatasetTransformations(datasetReq, dataset.id, transaction) + await this.createConnectorInstances(datasetReq, dataset.id, transaction) + await transaction.commit() + return dataset + } catch(err) { + transaction && await transaction.rollback() + throw err + } + } -} -const getDatasetDefaults = async (payload: Record): Promise> => { - const datasetPayload = mergeDatasetConfigs(defaultDatasetConfig, payload) - return datasetPayload -} + createDataset = async (datasetReq: Record, transaction: any): Promise> => { + const dataset = _.omit(datasetReq, ["transformations_config", "connector_config"]) + const mergedDataset = this.mergeDatasetConfigs(defaultDatasetConfig, dataset) + const data = { ...mergedDataset, version_key: Date.now().toString() } + const response = await DatasetDraft.create(data, {transaction: transaction}) + const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: data.version_key } + logger.info({ datasetReq, message: `Dataset Created Successfully with id:${_.get(response, ["dataValues", "id"])}`, response: responseData }) + return responseData + } -const setRedisDBConfig = async (datasetConfig: Record): Promise> => { - let nextRedisDB = datasetConfig.redis_db; - const { results }: any = await query("SELECT nextval('redis_db_index')") - if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; - return _.assign(datasetConfig, { "redis_db": nextRedisDB }) -} + mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { + const { id, dataset_id } = requestPayload; + const datasetId = !id ? dataset_id : id + const modifyPayload = { ...requestPayload, id: datasetId, router_config: { topic: datasetId } } + const defaults = _.cloneDeep(defaultConfig) + const datasetConfigs = _.merge(defaults, modifyPayload) + return datasetConfigs + } -const getMasterDatasetDefaults = async (payload: Record): Promise> => { - const masterDatasetPayload = mergeDatasetConfigs(defaultMasterConfig, payload) - let datasetConfig = masterDatasetPayload.dataset_config - datasetConfig = await setRedisDBConfig(datasetConfig); - return _.assign(masterDatasetPayload, datasetConfig); -} + createDatasetTransformations = async (dataset: Record, datasetId: string, transaction: any) => { + const transformationConfig: any = this.getTransformationConfig({ transformationPayload: _.get(dataset, "transformations_config"), datasetId: datasetId }) + if (!_.isEmpty(transformationConfig)) { + await DatasetTransformationsDraft.bulkCreate(transformationConfig, {transaction: transaction}); + logger.info({ transformationConfig, message: `Dataset transformations records created successsfully for dataset:${datasetId}` }) + } + } -const getDefaultHandler = (datasetType: string) => { - if (datasetType == DatasetType.Dataset) { - return getDatasetDefaults; - } else { - return getMasterDatasetDefaults; + createConnectorInstances = async (dataset: Record, datasetId: string, transaction: any) => { + + const connectorConfigs = _.get(dataset, "connectors_config") + if (!_.isEmpty(connectorConfigs)) { + const uniqueConnectors = _.uniqWith(connectorConfigs, (a: Record, b: Record) => { + return _.isEqual(a.connector_id, b.connector_id) && _.isEqual(a.connector_config, b.connector_config) + }) + const connectorsToPersist = _.map(uniqueConnectors, (config) => { + return { + id: v4(), + dataset_id: datasetId, + connector_id: config.connector_id, + connector_config: cipherService.encrypt(JSON.stringify(config.connector_config)), + operations_config: config.operations_config + } + }) + const v1DataSources = _.map(_.filter(connectorsToPersist, (conn) => { + return _.includes(["kafka-connector", "debezium-connector", "sb-knowlg-connector"], conn.connector_id) + }), (config) => { + return { + id: v4(), + dataset_id: datasetId, + connector_type: this.getConnectorTypeV1(config.connector_id), + connector_config: this.getConnectorConfigV1(config.connector_config), + operations_config: config.operations_config + } + }) + await ConnectorInstancesDraft.bulkCreate(connectorsToPersist, {transaction: transaction}); + await DatasetSourceConfigDraft.bulkCreate(v1DataSources, {transaction: transaction}); + logger.info({ uniqueConnectors, message: `Connector instances created successsfully for dataset:${datasetId}` }) + } } -} -export const getDefaultValue = async (payload: Record) => { - const datasetType = _.get(payload, "type"); - const getDatasetDefaults = getDefaultHandler(datasetType) - const datasetDefaults = await getDatasetDefaults(payload) - return _.omit(datasetDefaults, ["transformations_config"]) + getConnectorTypeV1 = (connector_id: string) : string => { + switch(connector_id) { + case "debezium-connector": + return "debezium" + case "sb-knowlg-connector": + return "neo4j" + default: + return "kafka" + } + } + + getConnectorConfigV1 = (connector_config: any): Record => { + return { + "topic": connector_config.source_kafka_topic, + "kafkaBrokers": connector_config.source_kafka_broker_servers + } + } + + getTransformationConfig = (configs: Record): Record => { + + const { transformationPayload, datasetId } = configs + if (transformationPayload) { + + let transformations: any = [] + const transformationFieldKeys = _.flatten(_.map(transformationPayload, fields => _.get(fields, ["field_key"]))) + const duplicateFieldKeys: Array = this.getDuplicateConfigs(transformationFieldKeys) + + if (!_.isEmpty(duplicateFieldKeys)) { + logger.info({ message: `Duplicate transformations provided by user are [${duplicateFieldKeys}]` }) + } + + _.forEach(transformationPayload, payload => { + const fieldKey = _.get(payload, "field_key") + const transformationExists = _.some(transformations, field => _.get(field, "field_key") == fieldKey) + if (!transformationExists) { + transformations = _.flatten(_.concat(transformations, { ...payload, id: `${datasetId}_${fieldKey}`, dataset_id: datasetId })) + } + }) + return transformations + } + return [] + } + + getDuplicateConfigs = (configs: Array) => { + return _.filter(configs, (item: string, index: number) => _.indexOf(configs, item) !== index); + } } -const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { - const { id, dataset_id } = requestPayload; - const datasetId = !id ? dataset_id : id - const modifyPayload = { ...requestPayload, id: datasetId, router_config: { topic: datasetId } } - const defaults = _.cloneDeep(defaultConfig) - const datasetConfigs = _.merge(defaults, modifyPayload) - return datasetConfigs -} \ No newline at end of file +export const datasetService = new DatasetService(); \ No newline at end of file From 3308eb5c5d20ab99aada35cca6c1bd29c48119c7 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 4 Jul 2024 12:20:11 +0530 Subject: [PATCH 006/235] feat 000: Dataset create API refactoring --- .../DatasetCreate/DatasetCreate.ts | 4 +- .../src/v2/models/ConnectorInstancesDraft.ts | 58 --------- api-service/src/v2/models/DatasetDraft.ts | 10 ++ api-service/src/v2/services/DatasetService.ts | 113 +++--------------- 4 files changed, 31 insertions(+), 154 deletions(-) delete mode 100644 api-service/src/v2/models/ConnectorInstancesDraft.ts diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index d75a03c4..7dcff1d0 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -1,3 +1,4 @@ +import _ from "lodash"; import { Request, Response } from "express"; import httpStatus from "http-status"; import logger from "../../logger"; @@ -57,8 +58,7 @@ const datasetCreate = async (req: Request, res: Response) => { if(!isRequestValid) { return; } - - const dataset = await datasetService.create(req.body.request); + const dataset = await datasetService.createDataset(req.body.request); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } diff --git a/api-service/src/v2/models/ConnectorInstancesDraft.ts b/api-service/src/v2/models/ConnectorInstancesDraft.ts deleted file mode 100644 index 7d7783c5..00000000 --- a/api-service/src/v2/models/ConnectorInstancesDraft.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { DataTypes } from "sequelize"; -import { sequelize } from "../connections/databaseConnection"; - -export const ConnectorInstancesDraft = sequelize.define("connector_instances_draft", { - id: { - type: DataTypes.STRING, - primaryKey: true, - allowNull: false, - autoIncrement: true - }, - dataset_id: { - type: DataTypes.STRING, - allowNull: false - }, - connector_id: { - type: DataTypes.STRING, - allowNull: false - }, - data_format: { - type: DataTypes.STRING, - defaultValue: "json", - allowNull: false - }, - connector_config: { - type: DataTypes.STRING, - allowNull: false - }, - operations_config: { - type: DataTypes.JSON, - defaultValue: {}, - allowNull: false - }, - status: { - type: DataTypes.ENUM("Draft", "Publishing", "Live", "Retired"), - defaultValue: "Draft", - allowNull: false - }, - validation_config: { - type: DataTypes.JSON, - defaultValue: {} - }, - created_by: { - type: DataTypes.STRING, - defaultValue: "SYSTEM" - }, - updated_by: { - type: DataTypes.STRING, - defaultValue: "SYSTEM" - }, - published_date: { - type: DataTypes.NUMBER - } -}, { - tableName: "connector_instances_draft", - timestamps: true, - createdAt: "created_date", - updatedAt: "updated_date" -}) \ No newline at end of file diff --git a/api-service/src/v2/models/DatasetDraft.ts b/api-service/src/v2/models/DatasetDraft.ts index 9892032f..b183289e 100644 --- a/api-service/src/v2/models/DatasetDraft.ts +++ b/api-service/src/v2/models/DatasetDraft.ts @@ -55,6 +55,16 @@ export const DatasetDraft = sequelize.define("datasets_draft", { allowNull: true, defaultValue: {} }, + transformations_config: { + type: DataTypes.JSON, + allowNull: true, + defaultValue: {} + }, + connectors_config: { + type: DataTypes.JSON, + allowNull: true, + defaultValue: {} + }, tags: { type: DataTypes.ARRAY(DataTypes.TEXT), allowNull: true, diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 30f79f15..fe313e8f 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -1,15 +1,12 @@ import _ from "lodash"; -import { v4 } from "uuid"; import logger from "../logger"; import { cipherService } from "./CipherService"; -import { sequelize } from "../connections/databaseConnection"; import { defaultDatasetConfig } from "../configs/DatasetConfigDefault"; import { Dataset } from "../models/Dataset"; import { DatasetDraft } from "../models/DatasetDraft"; import { DatasetTransformations } from "../models/Transformation"; -import { ConnectorInstancesDraft } from "../models/ConnectorInstancesDraft"; import { DatasetTransformationsDraft } from "../models/TransformationDraft"; -import { DatasetSourceConfigDraft } from "../models/DatasetSourceConfigDraft"; + class DatasetService { @@ -55,29 +52,19 @@ class DatasetService { return DatasetTransformations.findAll({ where: { dataset_id }, raw: true }); } - create = async (datasetReq: Record): Promise> => { - - let transaction; - try { - transaction = await sequelize.transaction() - const dataset = await this.createDataset(datasetReq, transaction); - await this.createDatasetTransformations(datasetReq, dataset.id, transaction) - await this.createConnectorInstances(datasetReq, dataset.id, transaction) - await transaction.commit() - return dataset - } catch(err) { - transaction && await transaction.rollback() - throw err - } - - } - - createDataset = async (datasetReq: Record, transaction: any): Promise> => { - const dataset = _.omit(datasetReq, ["transformations_config", "connector_config"]) + createDataset = async (datasetReq: Record): Promise> => { + const transformationsConfig:Array> = _.get(datasetReq, "transformations_config") + const connectorsConfig:Array> = _.get(datasetReq, "connectors_config") + const dataset = _.omit(datasetReq, ["transformations_config", "connectors_config"]) const mergedDataset = this.mergeDatasetConfigs(defaultDatasetConfig, dataset) - const data = { ...mergedDataset, version_key: Date.now().toString() } - const response = await DatasetDraft.create(data, {transaction: transaction}) - const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: data.version_key } + const draftDataset = { + ...mergedDataset, + version_key: Date.now().toString(), + transformations_config: this.getDatasetTransformations(transformationsConfig), + connectors_config: this.getDatasetConnectors(connectorsConfig), + } + const response = await DatasetDraft.create(draftDataset) + const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: draftDataset.version_key } logger.info({ datasetReq, message: `Dataset Created Successfully with id:${_.get(response, ["dataValues", "id"])}`, response: responseData }) return responseData } @@ -91,93 +78,31 @@ class DatasetService { return datasetConfigs } - createDatasetTransformations = async (dataset: Record, datasetId: string, transaction: any) => { - const transformationConfig: any = this.getTransformationConfig({ transformationPayload: _.get(dataset, "transformations_config"), datasetId: datasetId }) - if (!_.isEmpty(transformationConfig)) { - await DatasetTransformationsDraft.bulkCreate(transformationConfig, {transaction: transaction}); - logger.info({ transformationConfig, message: `Dataset transformations records created successsfully for dataset:${datasetId}` }) - } - } - - createConnectorInstances = async (dataset: Record, datasetId: string, transaction: any) => { + getDatasetConnectors = (connectorConfigs: Array>): Array> => { - const connectorConfigs = _.get(dataset, "connectors_config") if (!_.isEmpty(connectorConfigs)) { const uniqueConnectors = _.uniqWith(connectorConfigs, (a: Record, b: Record) => { return _.isEqual(a.connector_id, b.connector_id) && _.isEqual(a.connector_config, b.connector_config) }) - const connectorsToPersist = _.map(uniqueConnectors, (config) => { + return _.map(uniqueConnectors, (config) => { return { - id: v4(), - dataset_id: datasetId, connector_id: config.connector_id, connector_config: cipherService.encrypt(JSON.stringify(config.connector_config)), operations_config: config.operations_config } }) - const v1DataSources = _.map(_.filter(connectorsToPersist, (conn) => { - return _.includes(["kafka-connector", "debezium-connector", "sb-knowlg-connector"], conn.connector_id) - }), (config) => { - return { - id: v4(), - dataset_id: datasetId, - connector_type: this.getConnectorTypeV1(config.connector_id), - connector_config: this.getConnectorConfigV1(config.connector_config), - operations_config: config.operations_config - } - }) - await ConnectorInstancesDraft.bulkCreate(connectorsToPersist, {transaction: transaction}); - await DatasetSourceConfigDraft.bulkCreate(v1DataSources, {transaction: transaction}); - logger.info({ uniqueConnectors, message: `Connector instances created successsfully for dataset:${datasetId}` }) - } - } - - getConnectorTypeV1 = (connector_id: string) : string => { - switch(connector_id) { - case "debezium-connector": - return "debezium" - case "sb-knowlg-connector": - return "neo4j" - default: - return "kafka" - } - } - - getConnectorConfigV1 = (connector_config: any): Record => { - return { - "topic": connector_config.source_kafka_topic, - "kafkaBrokers": connector_config.source_kafka_broker_servers } + return [] } - getTransformationConfig = (configs: Record): Record => { + getDatasetTransformations = (configs: Array>): Array> => { - const { transformationPayload, datasetId } = configs - if (transformationPayload) { - - let transformations: any = [] - const transformationFieldKeys = _.flatten(_.map(transformationPayload, fields => _.get(fields, ["field_key"]))) - const duplicateFieldKeys: Array = this.getDuplicateConfigs(transformationFieldKeys) - - if (!_.isEmpty(duplicateFieldKeys)) { - logger.info({ message: `Duplicate transformations provided by user are [${duplicateFieldKeys}]` }) - } - - _.forEach(transformationPayload, payload => { - const fieldKey = _.get(payload, "field_key") - const transformationExists = _.some(transformations, field => _.get(field, "field_key") == fieldKey) - if (!transformationExists) { - transformations = _.flatten(_.concat(transformations, { ...payload, id: `${datasetId}_${fieldKey}`, dataset_id: datasetId })) - } - }) - return transformations + if (configs) { + return _.uniqBy(configs, "field_key") } return [] } - getDuplicateConfigs = (configs: Array) => { - return _.filter(configs, (item: string, index: number) => _.indexOf(configs, item) !== index); - } } export const datasetService = new DatasetService(); \ No newline at end of file From 1014d1d62bbce4abe9570d2d42e18e97918fd380 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 4 Jul 2024 16:37:01 +0530 Subject: [PATCH 007/235] #OBS-I115: Dataset create API refactoring --- .../DatasetCreate/DatasetCreate.ts | 53 +- .../DatasetCreateValidationSchema.json | 89 ++- .../DatasetUpdate/DatasetUpdate.ts | 525 ++++++------------ .../DatasetUpdateValidationSchema.json | 314 ++++++----- api-service/src/v2/services/DatasetService.ts | 65 +-- 5 files changed, 454 insertions(+), 592 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index 7dcff1d0..17e18943 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -7,6 +7,8 @@ import DatasetCreate from "./DatasetCreateValidationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { ErrorObject } from "../../types/ResponseModel"; +import { cipherService } from "../../services/CipherService"; +import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; export const apiId = "api.datasets.create" @@ -58,8 +60,57 @@ const datasetCreate = async (req: Request, res: Response) => { if(!isRequestValid) { return; } - const dataset = await datasetService.createDataset(req.body.request); + const draftDataset = getDraftDataset(req.body.request) + const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } +const getDraftDataset = (datasetReq: Record): Record => { + const transformationsConfig:Array> = _.get(datasetReq, "transformations_config") + const connectorsConfig:Array> = _.get(datasetReq, "connectors_config") + const dataset = _.omit(datasetReq, ["transformations_config", "connectors_config"]) + const mergedDataset = mergeDatasetConfigs(defaultDatasetConfig, dataset) + const draftDataset = { + ...mergedDataset, + version_key: Date.now().toString(), + transformations_config: getDatasetTransformations(transformationsConfig), + connectors_config: getDatasetConnectors(connectorsConfig), + } + return draftDataset; +} + +const mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { + const { id, dataset_id } = requestPayload; + const datasetId = !id ? dataset_id : id + const modifyPayload = { ...requestPayload, id: datasetId, router_config: { topic: datasetId } } + const defaults = _.cloneDeep(defaultConfig) + const datasetConfigs = _.merge(defaults, modifyPayload) + return datasetConfigs +} + +const getDatasetConnectors = (connectorConfigs: Array>): Array> => { + + if (!_.isEmpty(connectorConfigs)) { + const uniqueConnectors = _.uniqWith(connectorConfigs, (a: Record, b: Record) => { + return _.isEqual(a.connector_id, b.connector_id) && _.isEqual(a.connector_config, b.connector_config) + }) + return _.map(uniqueConnectors, (config) => { + return { + connector_id: config.connector_id, + connector_config: cipherService.encrypt(JSON.stringify(config.connector_config)), + operations_config: config.operations_config + } + }) + } + return [] +} + +const getDatasetTransformations = (configs: Array>): Array> => { + + if (configs) { + return _.uniqBy(configs, "field_key") + } + return [] +} + export default datasetCreate; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index f12eb65d..b6810c60 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -56,7 +56,8 @@ "type": "object", "properties": { "is_batch_event": { - "type": "boolean" + "type": "boolean", + "default": true }, "extraction_key": { "type": "string", @@ -120,46 +121,6 @@ "required": ["$schema", "type", "properties", "additionalProperties"], "additionalProperties": false }, - "denorm_config": { - "type": "object", - "properties": { - "denorm_fields": { - "type": "array", - "items": { - "type": "object", - "properties": { - "denorm_dataset": { - "type": "string", - "minLength": 1 - }, - "denorm_key": { - "type": "string", - "minLength": 1 - }, - "denorm_out_field": { - "type": "string", - "minLength": 1 - }, - "jsonata_expr": { - "type": "string", - "minLength": 1 - } - }, - "oneOf": [ - { - "required": ["denorm_dataset", "denorm_out_field", "denorm_key"] - }, - { - "required": ["denorm_dataset", "denorm_out_field", "jsonata_expr"] - } - ], - "additionalProperties": false - } - } - }, - "required": ["denorm_fields"], - "additionalProperties": false - }, "dataset_config": { "type": "object", "properties": { @@ -269,13 +230,49 @@ } }, "additionalProperties": false, - "required": [ - "field_key", - "transformation_function", - "mode" - ] + "required": ["field_key", "transformation_function", "mode"] } }, + "denorm_config": { + "type": "object", + "properties": { + "denorm_fields": { + "type": "array", + "items": { + "type": "object", + "properties": { + "denorm_dataset": { + "type": "string", + "minLength": 1 + }, + "denorm_key": { + "type": "string", + "minLength": 1 + }, + "denorm_out_field": { + "type": "string", + "minLength": 1 + }, + "jsonata_expr": { + "type": "string", + "minLength": 1 + } + }, + "oneOf": [ + { + "required": ["denorm_dataset", "denorm_out_field", "denorm_key"] + }, + { + "required": ["denorm_dataset", "denorm_out_field", "jsonata_expr"] + } + ], + "additionalProperties": false + } + } + }, + "required": ["denorm_fields"], + "additionalProperties": false + }, "connectors_config": { "type": "array", "items": { diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts index 24e16baf..e7aa73c7 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts @@ -1,418 +1,207 @@ -import httpStatus from "http-status"; import { Request, Response } from "express"; +import httpStatus from "http-status"; +import logger from "../../logger"; +import _ from "lodash"; +import Model from "sequelize/types/model"; +import { ErrorObject } from "../../types/ResponseModel"; +import { DatasetStatus } from "../../types/DatasetModels"; import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { cipherService } from "../../services/CipherService"; +import { datasetService } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import DatasetUpdate from "./DatasetUpdateValidationSchema.json"; -import _ from "lodash"; -import { DatasetStatus } from "../../types/DatasetModels"; -import { ErrorObject } from "../../types/ResponseModel"; -import { DatasetDraft } from "../../models/DatasetDraft"; -import logger from "../../logger"; -import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { generateDataSource, getDefaultValue, getDraftTransformations, getDuplicateConfigs, setReqDatasetId } from "../../services/DatasetService"; -import { DatasourceDraft } from "../../models/DatasourceDraft"; -import { ingestionConfig } from "../../configs/IngestionConfig"; export const apiId = "api.datasets.update"; export const invalidInputErrCode = "DATASET_UPDATE_INPUT_INVALID" export const errorCode = "DATASET_UPDATE_FAILURE" -const datasetUpdate = async (req: Request, res: Response) => { - const requestBody = req.body - const msgid = _.get(req, ["body", "params", "msgid"]); - const resmsgid = _.get(res, "resmsgid"); - try { - const datasetId = _.get(req, ["body", "request", "dataset_id"]) - setReqDatasetId(req, datasetId) +const isValidRequest = async (req: Request, res: Response): Promise => { - const isRequestValid: Record = schemaValidation(req.body, DatasetUpdate) - if (!isRequestValid.isValid) { - logger.error({ code: invalidInputErrCode, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code: invalidInputErrCode, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } + const isRequestValid: Record = schemaValidation(req.body, DatasetUpdate) + if (!isRequestValid.isValid) { + logger.error({ code: "DATASET_UPDATE_INPUT_INVALID", apiId, body: req.body, message: isRequestValid.message }) + ResponseHandler.errorResponse({ + code: "DATASET_UPDATE_INPUT_INVALID", + message: isRequestValid.message, + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + return false; + } + + const datasetBody = req.body.request + const { dataset_id, version_key, ...rest } = datasetBody + if (_.isEmpty(rest)) { + const code = "DATASET_UPDATE_NO_FIELDS" + logger.error({ code, apiId, body: req.body, message: `Provide atleast one field in addition to the dataset_id:${dataset_id} and version_key:${version_key} to update the dataset` }) + ResponseHandler.errorResponse({ + code, + message: "Provide atleast one field in addition to the dataset_id to update the dataset", + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + return false; + } - const datasetBody = req.body.request - const { dataset_id, version_key, ...rest } = datasetBody - if (_.isEmpty(rest)) { - const code = "DATASET_UPDATE_NO_FIELDS" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Provide atleast one field in addition to the dataset_id:${dataset_id} and version_key:${version_key} to update the dataset` }) - return ResponseHandler.errorResponse({ - code, - message: "Provide atleast one field in addition to the dataset_id to update the dataset", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } + const duplicateDenormKeys = datasetService.getDuplicateDenormKey(_.get(req, ["body", "request", "denorm_config"])) + if (!_.isEmpty(duplicateDenormKeys)) { + const code = "DATASET_DUPLICATE_DENORM_KEY" + logger.error({ code: "DATASET_DUPLICATE_DENORM_KEY", body: req.body, message: `Duplicate denorm output fields found. Duplicate Denorm out fields are [${duplicateDenormKeys}]` }) + ResponseHandler.errorResponse({ + code: "DATASET_DUPLICATE_DENORM_KEY", + statusCode: 400, + message: "Duplicate denorm key found", + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + return false; + } + + return true; +} - const { isDatasetExists, datasetStatus, invalidVersionKey, validVersionKey } = await checkDatasetExists(dataset_id, version_key); - if (!isDatasetExists) { - const code = "DATASET_NOT_EXISTS" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Dataset does not exists with id:${dataset_id}` }) - return ResponseHandler.errorResponse({ - code, - message: "Dataset does not exists to update", +const isValidDataset = (datasetModel: Model | null, req: Request, res: Response) : boolean => { + + const datasetId = _.get(req, ["body", "request", "dataset_id"]) + const versionKey = _.get(req, ["body", "request", "version_key"]) + if(datasetModel) { + const dataset:Record = datasetModel.toJSON + if(dataset.api_version !== "v2") { + logger.error({ code: "DATASET_API_VERSION_MISMATCH", apiId, body: req.body, message: `Draft dataset api version is not v2:${datasetId}` }) + ResponseHandler.errorResponse({ + code: "DATASET_API_VERSION_MISMATCH", + message: "Draft dataset api version is not v2", statusCode: 404, errCode: "NOT_FOUND" } as ErrorObject, req, res); - } - - if (isDatasetExists && !_.includes([DatasetStatus.Draft, DatasetStatus.ReadyToPublish], datasetStatus)) { - const code = "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Dataset with id:${dataset_id} cannot be updated as it is not in draft state` }) - return ResponseHandler.errorResponse({ - code, - message: "Dataset cannot be updated as it is not in draft state", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - if (invalidVersionKey) { - const code = "DATASET_OUTDATED" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `The dataset:${dataset_id} with version_key:${version_key} is outdated. Please try to fetch latest changes of the dataset with version key:${validVersionKey} and perform the updates` }) - return ResponseHandler.errorResponse({ - code, + return false; + } + if(dataset.version_key !== versionKey) { + logger.error({ code: "DATASET_OUTDATED", body: req.body, message: `The dataset:${datasetId} with version_key:${versionKey} is outdated. Please try to fetch latest changes of the dataset with version key:${dataset.version_key} and perform the updates` }) + ResponseHandler.errorResponse({ + code: "DATASET_OUTDATED", message: "The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates", statusCode: 409, errCode: "CONFLICT" } as ErrorObject, req, res); + return false; } - - const duplicateDenormKeys = getDuplicateDenormKey(_.get(datasetBody, "denorm_config")) - if (!_.isEmpty(duplicateDenormKeys)) { - const code = "DATASET_DUPLICATE_DENORM_KEY" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `Duplicate denorm output fields found. Duplicate Denorm out fields are [${duplicateDenormKeys}]` }) - return ResponseHandler.errorResponse({ - code, + if(!_.includes([DatasetStatus.Draft, DatasetStatus.ReadyToPublish], dataset.status)) { + logger.error({ code: "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE", body: req.body, message: `Dataset with id:${datasetId} cannot be updated as it is not in draft state` }) + ResponseHandler.errorResponse({ + code: "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE", + message: "Dataset cannot be updated as it is not in draft state", statusCode: 400, - message: `Dataset contains duplicate denorm out keys:[${duplicateDenormKeys}]`, errCode: "BAD_REQUEST" } as ErrorObject, req, res); + return false; } - - const updatedDatasetConfigs = await getDatasetUpdatedConfigs(datasetBody); - const datasetPayload = await mergeExistingDataset(updatedDatasetConfigs) - const updatedPayload = await getDefaultValue(datasetPayload) - logger.debug({ datasetPayload }) - const transformationConfigs = _.get(datasetBody, "transformation_config") - await manageTransformations(transformationConfigs, dataset_id); - - const { data_schema } = datasetBody - if (data_schema) { - const { transformation_config } = datasetBody - const { dataset_config, id, dataset_id, denorm_config } = updatedPayload - const datasourcePayload = await generateDataSource({ indexCol: _.get(dataset_config, ["timestamp_key"]), transformation_config, denorm_config, data_schema, id, dataset_id, action: "edit" }) - await DatasourceDraft.update(datasourcePayload, { where: { id: _.get(datasourcePayload, "id") }}) - } - - await DatasetDraft.update(updatedPayload, { where: { id: dataset_id }}) - - const responsedata = { message: "Dataset is updated successfully", id: dataset_id, version_key: _.get(datasetPayload, "version_key") } - logger.info({ apiId, msgid, requestBody, resmsgid, message: `Dataset updated successfully with id:${dataset_id}`, response: responsedata }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responsedata }); - } catch (error: any) { - const code = _.get(error, "code") || errorCode - logger.error({ ...error, apiId, code, msgid, requestBody, resmsgid }) - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code, message: "Failed to update dataset" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } -} - -const manageTransformations = async (transformations: Record, datasetId: string) => { - if (transformations) { - const transformationConfigs = await getTransformationConfigs(transformations, datasetId); - if (transformationConfigs) { - const { addTransformation, updateTransformation, deleteTransformation } = transformationConfigs; - - if (!_.isEmpty(addTransformation)) { - await DatasetTransformationsDraft.bulkCreate(addTransformation); - } - - if (!_.isEmpty(updateTransformation)) { - await Promise.all(updateTransformation.map((record: any) => DatasetTransformationsDraft.update(record, { where: { id: record.id }}))); - } - - if (!_.isEmpty(deleteTransformation)) { - await DatasetTransformationsDraft.destroy({ where: { id: deleteTransformation }}); - } - } - } -} - -const checkDatasetExists = async (dataset_id: string, version_key: string): Promise> => { - const datasetExists: Record | null = await getExistingDataset(dataset_id) - if (datasetExists) { - const validVersionKey = _.get(datasetExists, "version_key") - const apiVersion = _.get(datasetExists, "api_version") - if (validVersionKey !== version_key && apiVersion) { - return { isDatasetExists: true, datasetStatus: datasetExists.status, invalidVersionKey: true, validVersionKey } - } - return { isDatasetExists: true, datasetStatus: datasetExists.status } + return true; } else { - return { isDatasetExists: false, datasetStatus: "" } - } -} - -const getDatasetUpdatedConfigs = async (payload: Record): Promise> => { - const { validation_config, extraction_config, dedup_config, tags, denorm_config, dataset_id } = payload - const existingDataset: any = await getExistingDataset(dataset_id) - const updatedConfigs = { - validation_config: validation_config ? setValidationConfigs(validation_config) : null, - extraction_config: extraction_config ? setExtractionConfigs(extraction_config) : null, - dedup_config: dedup_config ? setDedupConfigs(dedup_config) : null, - tags: tags ? getUpdatedTags(tags, _.get(existingDataset, "tags")) : null, - denorm_config: denorm_config ? setDenormConfigs(denorm_config, _.get(existingDataset, ["denorm_config"])) : null, - version_key: Date.now().toString(), - status: _.isEmpty(tags) ? DatasetStatus.Draft : null + logger.error({ code: "DATASET_NOT_EXISTS", body: req.body, message: `Dataset does not exists with id:${datasetId}` }) + ResponseHandler.errorResponse({ + code: "DATASET_NOT_EXISTS", + message: "Dataset does not exists to update", + statusCode: 404, + errCode: "NOT_FOUND" + } as ErrorObject, req, res); + return false; } - return _.pickBy({ ...payload, ...updatedConfigs }, _.identity) } -const getDuplicateDenormKey = (denormConfig: Record): Array => { - if (denormConfig && _.isArray(_.get(denormConfig, "denorm_fields"))) { - const denormFields = _.get(denormConfig, "denorm_fields") - const denormOutKeys = _.compact(_.map(denormFields, field => _.get(field, "action") == "add" && _.get(field, "values.denorm_out_field"))) - const duplicateDenormKeys: Array = _.filter(denormOutKeys, (item: string, index: number) => _.indexOf(denormOutKeys, item) !== index); - return duplicateDenormKeys; +const datasetUpdate = async (req: Request, res: Response) => { + + const isRequestValid = await isValidRequest(req, res) + if(!isRequestValid) { + return; } - return [] -} -const setValidationConfigs = (configs: Record): Record => { - const validationStatus = _.get(configs, "validate") - if (!validationStatus) { - _.set(configs, "mode", defaultDatasetConfig.validation_config.mode) + const datasetReq = req.body.request; + const datasetModel = await datasetService.getDraftDataset(datasetReq.dataset_id) + if(!isValidDataset(datasetModel, req, res)) { + return; } - return configs; + + const draftDataset = mergeDraftDataset(datasetModel, datasetReq); + const response = datasetService.updateDraftDataset(draftDataset); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } -const setExtractionConfigs = (configs: Record): Record => { - const isBatchEvent = _.get(configs, "is_batch_event") - if (!isBatchEvent) { - return _.merge(configs, defaultDatasetConfig.extraction_config) - } - return configs +const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { + + let dataset:Record = { + version_key: datasetReq.version_key, + name: datasetReq.name || _.get(datasetModel, ["name"]) + } + if(datasetReq.validation_config) dataset["validation_config"] = datasetReq.validation_config + if(datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config + if(datasetReq.dedup_config) dataset["dedup_config"] = datasetReq.dedup_config + if(datasetReq.data_schema) dataset["data_schema"] = datasetReq.data_schema + if(datasetReq.dataset_config) dataset["dataset_config"] = datasetReq.dataset_config + if(datasetReq.transformations_config) + dataset["transformations_config"] = mergeTransformationsConfig(_.get(datasetModel, ["transformations_config"]), datasetReq.transformations_config) + if(datasetReq.denorm_config) dataset["denorm_config"] = mergeDenormConfig(_.get(datasetModel, ["denorm_config"]), datasetReq.denorm_config) + if(datasetReq.connectors_config) dataset["connectors_config"] = mergeConnectorsConfig(_.get(datasetModel, ["connectors_config"]), datasetReq.connectors_config) + if(datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) + if(datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data + + return dataset; } -const setDedupConfigs = (configs: Record): Record => { - const dropDuplicates = _.get(configs, "drop_duplicates") - if (!dropDuplicates) { - return { ...defaultDatasetConfig.dedup_config, drop_duplicates: dropDuplicates } - } - return configs +const mergeTransformationsConfig = (currentConfigs: any, newConfigs: any) => { + const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.field_key") + const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + + return _.unionWith( + addConfigs, + _.reject(currentConfigs, (config) => { return _.includes(removeConfigs, config.field_key)}), + (a, b) => { + return a.field_key === b.field_key + } + ) } -const getUpdatedTags = (newTagsPayload: Record, datasetTags: Array): Record => { - - const getTagsPayload = (action: string) => { - return _.compact(_.flatten(_.map(newTagsPayload, fields => { - if (fields.action == action) return _.map(fields.values, value => _.toLower(value)) - }))) - } - const tagsToAdd = getTagsPayload("add") - const tagsToRemove = getTagsPayload("remove") - - const duplicateTagsToAdd: Array = getDuplicateConfigs(tagsToAdd) - if (!_.isEmpty(duplicateTagsToAdd)) { - logger.info({ apiId, message: `Duplicate tags provided by user to add are [${duplicateTagsToAdd}]` }) - } - - const duplicateTagsToRemove: Array = getDuplicateConfigs(tagsToRemove) - if (!_.isEmpty(duplicateTagsToRemove)) { - logger.info({ apiId, message: `Duplicate tags provided by user to remove are [${duplicateTagsToRemove}]` }) - } +const mergeDenormConfig = (currentConfig: any, newConfig: any) => { - const checkTagToAdd = _.intersection(datasetTags, _.uniq(tagsToAdd)) - if (_.size(checkTagToAdd)) { - throw { - code: "DATASET_TAGS_EXISTS", - message: "Dataset tags already exist", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject - } + const removeConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "remove"}), "value.denorm_out_field") + const addConfigs = _.map(_.filter(newConfig.denorm_fields, {action: "upsert"}), "value") - const checkTagToRemove = _.intersection(datasetTags, _.uniq(tagsToRemove)) - if (_.size(checkTagToRemove) !== _.size(_.uniq(tagsToRemove))) { - throw { - code: "DATASET_TAGS_DO_NOT_EXIST", - message: "Dataset tags do not exist to remove", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject + const denormFields = _.unionWith( + addConfigs, + _.reject(currentConfig.denorm_fields, (config) => { return _.includes(removeConfigs, config.denorm_out_field)}), + (a, b) => { + return a.denorm_out_field === b.denorm_out_field + } + ) + return { + denorm_fields: denormFields } - - const tagsAfterAddition = _.concat(datasetTags, tagsToAdd) - const updatedTags = _.difference(tagsAfterAddition, tagsToRemove) - return _.flatten(updatedTags) } -const getTransformationConfigs = async (newTransformationPayload: Record, dataset_id: string): Promise> => { - const datasetTransformations: any = await getDraftTransformations(dataset_id) - let addTransformation: Record = [] - let updateTransformation: Record = [] - let deleteTransformation: Array = [] - - const existingTransformationsFields = _.map(datasetTransformations, config => config?.field_key) - const transformationFieldKeys = _.flatten(_.map(newTransformationPayload, fields => { - if (fields.action == "add") return fields.values.field_key - })) - - const duplicateFieldKeys: Array = getDuplicateConfigs(transformationFieldKeys) - if (!_.isEmpty(duplicateFieldKeys)) { - logger.info({ apiId, message: `Duplicate transformations provided by user are [${duplicateFieldKeys}]` }) - } - - const getTransformationPayload = (action: string) => { - return _.compact(_.flatten(_.map(newTransformationPayload, payload => { - if (payload.action == action) return payload.values - }))) - } - const transformationsToAdd = getTransformationPayload("add") - const transformationsToUpdate = getTransformationPayload("update") - const transformationsToRemove = getTransformationPayload("remove") - - const checkTransformations = (transformationPayload: Record) => { - return _.intersection(existingTransformationsFields, _.map(transformationPayload, payload => _.get(payload, ["field_key"]))) - } - const checkTransformationToAdd = checkTransformations(transformationsToAdd) - const checkTransformationToRemove = checkTransformations(transformationsToRemove) - const checkTransformationToUpdate = checkTransformations(transformationsToUpdate) +const mergeConnectorsConfig = (currConfigs: any, newConfigs: any) => { - if (_.size(checkTransformationToAdd)) { - throw { - code: "DATASET_TRANSFORMATIONS_EXIST", - message: "Dataset transformations already exists", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject - } + const removeConfigs = _.map(_.filter(newConfigs, {action: "remove"}), "value.connector_id") + const addConfigs = _.map(_.filter(newConfigs, {action: "upsert"}), "value") - const transformationExistsToUpdate = _.every(transformationsToUpdate, payload => _.includes(checkTransformationToUpdate, payload.field_key)) - if (!transformationExistsToUpdate) { - throw { - code: "DATASET_TRANSFORMATIONS_DO_NOT_EXIST", - message: "Dataset transformations do not exist to update", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject - } - - const isTransformationExistsToRemove = _.every(transformationsToRemove, payload => _.includes(checkTransformationToRemove, payload.field_key)) - if (!isTransformationExistsToRemove) { - throw { - code: "DATASET_TRANSFORMATIONS_DO_NOT_EXIST", - message: "Dataset transformations do not exist to remove", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject - } - - _.forEach(newTransformationPayload, fields => { - const { values, action } = fields - const fieldKey = _.get(values, "field_key") - if (action == "remove") { - const deleteFieldKey: string = `${dataset_id}_${fieldKey}` - deleteTransformation = _.uniq(_.flatten(_.concat(deleteTransformation, deleteFieldKey))) - } - if (action == "add") { - const transformationExists = _.some(addTransformation, field => field.field_key == fieldKey) - if (!transformationExists) { - addTransformation = _.flatten(_.concat(addTransformation, { ...values, id: `${dataset_id}_${fieldKey}`, dataset_id })) + return _.unionWith( + _.map(addConfigs, (config) => { + return { + connector_id: config.connector_id, + connector_config: cipherService.encrypt(JSON.stringify(config.connector_config)), + operations_config: config.operations_config } - } - if (action == "update") { - const existingTransformations = _.filter(datasetTransformations, transformationField => transformationField.field_key == fieldKey) - const updatedTransformations = _.merge(_.get(existingTransformations, "[0]"), values) - updateTransformation = _.flatten(_.concat(updateTransformation, updatedTransformations)) - } - }) - return { addTransformation, deleteTransformation, updateTransformation } + }), + _.reject(currConfigs, (config) => { return _.includes(removeConfigs, config.connector_id)}), + (a, b) => { + return a.connector_id === b.connector_id + } + ) } -const setDenormConfigs = (newDenormPayload: Record, datasetDenormConfigs: Record): Record => { - const datasetDenormFieldsKeys = _.map(_.get(datasetDenormConfigs, "denorm_fields"), fields => fields?.denorm_out_field) - const { denorm_fields } = newDenormPayload; - const existingDenormFields = _.get(datasetDenormConfigs, "denorm_fields") || [] - - if (_.isEmpty(denorm_fields)) { - return { denorm_fields } - } - - const getDenormPayload = (action: string) => { - return _.compact(_.flatten(_.map(denorm_fields, payload => { - if (payload.action == action) return payload.values - }))) - } - const denormsToAdd = getDenormPayload("add") - const denormsToRemove = getDenormPayload("remove") - - const denormFieldsToRemove = _.map(denormsToRemove, field => field.denorm_out_field) - const duplicateFieldKeys: Array = getDuplicateConfigs(denormFieldsToRemove) - if (!_.isEmpty(duplicateFieldKeys)) { - logger.info({ apiId, message: `Duplicate denorm out fields provided by user are [${duplicateFieldKeys}]` }) - } - - const checkDenormKeys = (denormKeys: Record) => { - return _.intersection(datasetDenormFieldsKeys, _.map(denormKeys, payload => _.get(payload, ["denorm_out_field"]))) - } - const checkDenormsToAdd = checkDenormKeys(denormsToAdd) - const checkDenormsToRemove = checkDenormKeys(denormsToRemove) - - if (_.size(checkDenormsToAdd)) { - throw { - code: "DATASET_DENORM_EXISTS", - message: "Denorm fields already exist", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject - } - - const isDenormExists = _.every(denormsToRemove, payload => _.includes(checkDenormsToRemove, payload.denorm_out_field)) - if (!isDenormExists) { - throw { - code: "DATASET_DENORM_DO_NOT_EXIST", - message: "Denorm fields do not exist to remove", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject - } - - const denormsFields = _.concat(existingDenormFields, denormsToAdd) - const updatedDenormFields = _.filter(denormsFields, fields => !_.includes(checkDenormsToRemove, fields.denorm_out_field)) - return { ...datasetDenormConfigs, denorm_fields: updatedDenormFields } -} - -const mergeExistingDataset = async (configs: Record): Promise> => { - const existingDataset = await getExistingDataset(_.get(configs, "dataset_id")) - const existingConfigs = _.omit(existingDataset, ["data_schema"]) - const mergedData = _.mergeWith(existingConfigs, configs, (existingValue, newValue) => { - if (_.isArray(existingValue) && _.isEmpty(newValue)) { - return []; - } - if (_.isArray(existingValue) && !_.isEmpty(newValue)) { - return newValue - } - }); - mergedData.api_version = "v2"; - if (_.isEmpty(_.get(mergedData, ["dataset_config", "timestamp_key"]))) { - _.set(mergedData, "dataset_config", { "timestamp_key": ingestionConfig.indexCol["Event Arrival Time"] }) - } - return _.omit(mergedData, ["dataset_id", "transformation_config", "created_date", "published_date"]) -} +const mergeTags = (currentTags: any, newConfigs: any) => { -export const getExistingDataset = async (id: string) => { - return DatasetDraft.findOne({ where: { id }, raw: true }) + const tagsToRemove = _.map(_.filter(newConfigs, {action: "remove"}), "value") + const tagsToAdd = _.map(_.filter(newConfigs, {action: "upsert"}), "value") + return _.union(_.pullAll(currentTags, tagsToRemove), tagsToAdd) } -export default datasetUpdate; +export default datasetUpdate; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 76f1bbb1..95ef5ee5 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -36,9 +36,6 @@ "type": "string", "minLength": 1 }, - "client_state": { - "type": "object" - }, "validation_config": { "type": "object", "properties": { @@ -50,18 +47,8 @@ "enum": ["Strict", "IgnoreNewFields"] } }, - "additionalProperties": false, - "required": ["validate"], - "if": { - "properties": { - "validate": { - "const": true - } - } - }, - "then": { - "required": ["mode"] - } + "required": ["validate", "mode"], + "additionalProperties": false }, "extraction_config": { "type": "object", @@ -85,71 +72,27 @@ "type": "string" } }, - "if": { - "properties": { - "drop_duplicates": { - "const": true - } - } - }, - "then": { - "properties": { - "dedup_key": { - "minLength": 1 - } - }, - "required": ["dedup_key"] - }, + "required": ["drop_duplicates", "dedup_key"], "additionalProperties": false } }, - "additionalProperties": false, - "required": ["is_batch_event"], - "if": { - "properties": { - "is_batch_event": { - "const": true - } - } - }, - "then": { - "properties": { - "extraction_key": { - "minLength": 1 - }, - "batch_id": { - "minLength": 1 - } - }, - "required": ["extraction_key", "dedup_config"] - } + "required": ["is_batch_event", "extraction_key", "dedup_config"], + "additionalProperties": false }, "dedup_config": { "type": "object", "properties": { "drop_duplicates": { - "type": "boolean" + "type": "boolean", + "default": true }, "dedup_key": { - "type": "string" - } - }, - "additionalProperties": false, - "if": { - "properties": { - "drop_duplicates": { - "const": true - } + "type": "string", + "minLength": 1 } }, - "then": { - "properties": { - "dedup_key": { - "minLength": 1 - } - }, - "required": ["dedup_key"] - } + "required": ["drop_duplicates", "dedup_key"], + "additionalProperties": false }, "data_schema": { "type": "object", @@ -180,37 +123,63 @@ "dataset_config": { "type": "object", "properties": { - "data_key": { - "type": "string" - }, - "timestamp_key": { - "type": "string" - }, - "mergedEvent": { - "type": "object" - }, - "configurations": { - "type": "object" - }, - "dataMappings": { - "type": "object" - }, "file_upload_path": { "type": "array", "items": { "type": "string", "minLength": 1 } + }, + "dataset_tz": { + "type": "string" + }, + "indexing_config": { + "type": "object", + "properties": { + "olap_store_enabled": { + "type": "boolean", + "default": false + }, + "lakehouse_enabled": { + "type": "boolean", + "default": true + }, + "cache_enabled": { + "type": "boolean", + "default": false + } + }, + "additionalProperties": false + }, + "keys_config": { + "type": "object", + "properties": { + "data_key": { + "type": "string" + }, + "partition_key": { + "type": "string" + }, + "timestamp_key": { + "type": "string" + }, + "timestamp_format": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false } }, + "required": ["indexing_config", "keys_config"], "additionalProperties": false }, - "transformation_config": { + "transformations_config": { "type": "array", "items": { "type": "object", "properties": { - "values": { + "value": { "type": "object", "properties": { "field_key": { @@ -218,63 +187,76 @@ "minLength": 1 }, "transformation_function": { - "type": "object" + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + }, + "condition": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "expr"], + "additionalProperties": false + }, + "datatype": { + "type": "string" + }, + "category": { + "type": "string", + "enum": ["pii", "transform", "derived"] + } + }, + "required": ["type", "expr"], + "additionalProperties": false }, "mode": { "type": "string", - "enum": ["Strict", "Lenient"] - }, - "metadata": { - "type": "object" + "enum": [ + "Strict", + "Lenient" + ] } }, - "additionalProperties": false, - "required": ["field_key"] + "required": ["field_key"], + "additionalProperties": false }, "action": { "type": "string", - "enum": ["add", "remove", "update"] + "enum": ["upsert", "remove"] } }, "additionalProperties": false, - "required": ["values", "action"], + "required": ["value", "action"], "if": { "properties": { "action": { - "const": "add" + "const": "upsert" } } }, "then": { "properties": { - "values": { + "value": { "required": ["transformation_function", "mode"] } } } } }, - "tags": { - "type": "array", - "items": { - "type": "object", - "properties": { - "values": { - "type": "array", - "items": { - "type": "string", - "minLength": 1 - }, - "minItems": 1 - }, - "action": { - "type": "string", - "enum": ["add", "remove"] - } - }, - "required": ["values", "action"] - } - }, "denorm_config": { "type": "object", "properties": { @@ -283,9 +265,13 @@ "items": { "type": "object", "properties": { - "values": { + "value": { "type": "object", "properties": { + "denorm_dataset": { + "type": "string", + "minLength": 1 + }, "denorm_key": { "type": "string", "minLength": 1 @@ -293,37 +279,115 @@ "denorm_out_field": { "type": "string", "minLength": 1 + }, + "jsonata_expr": { + "type": "string", + "minLength": 1 } }, - "required": ["denorm_out_field"] + "additionalProperties": false }, "action": { "type": "string", - "enum": ["add", "remove"] + "enum": ["upsert", "remove"] } }, "additionalProperties": false, - "required": ["values", "action"], + "required": ["value", "action"], "if": { "properties": { "action": { - "const": "add" + "enum": ["upsert", "update"] } } }, "then": { "properties": { - "values": { - "required": ["denorm_key"] + "value": { + "oneOf": [ + { + "required": ["denorm_dataset", "denorm_out_field", "denorm_key"] + }, + { + "required": ["denorm_dataset", "denorm_out_field", "jsonata_expr"] + } + ] } } } } } } + }, + "connectors_config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "type": "object", + "properties": { + "connector_id": { + "type": "string", + "minLength": 1 + }, + "connector_config": { + "type": "object" + }, + "operations_config": { + "type": "object" + } + }, + "required": ["connector_id"], + "additionalProperties": false + }, + "action": { + "type": "string", + "enum": ["upsert", "remove"] + } + }, + "additionalProperties": false, + "required": ["value", "action"], + "if": { + "properties": { + "action": { + "const": "upsert" + } + } + }, + "then": { + "properties": { + "value": { + "required": ["connector_config"] + } + } + } + } + }, + "tags": { + "type": "array", + "items": { + "type": "object", + "properties": { + "value": { + "type": "string", + "minLength": 1 + }, + "action": { + "type": "string", + "enum": ["upsert", "remove"] + } + }, + "required": ["value", "action"] + } + }, + "sample_data": { + "type": "object" } }, "required": ["dataset_id", "version_key"] } - } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false } diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index fe313e8f..c40f07c6 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -11,13 +11,7 @@ import { DatasetTransformationsDraft } from "../models/TransformationDraft"; class DatasetService { getDataset = async (datasetId: string, raw = false): Promise => { - const dataset = await Dataset.findOne({ - where: { - id: datasetId, - }, - raw: raw - }); - return dataset + return Dataset.findOne({ where: { id: datasetId }, raw: raw }); } getDuplicateDenormKey = (denormConfig: Record): Array => { @@ -52,57 +46,24 @@ class DatasetService { return DatasetTransformations.findAll({ where: { dataset_id }, raw: true }); } - createDataset = async (datasetReq: Record): Promise> => { - const transformationsConfig:Array> = _.get(datasetReq, "transformations_config") - const connectorsConfig:Array> = _.get(datasetReq, "connectors_config") - const dataset = _.omit(datasetReq, ["transformations_config", "connectors_config"]) - const mergedDataset = this.mergeDatasetConfigs(defaultDatasetConfig, dataset) - const draftDataset = { - ...mergedDataset, - version_key: Date.now().toString(), - transformations_config: this.getDatasetTransformations(transformationsConfig), - connectors_config: this.getDatasetConnectors(connectorsConfig), - } - const response = await DatasetDraft.create(draftDataset) - const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: draftDataset.version_key } - logger.info({ datasetReq, message: `Dataset Created Successfully with id:${_.get(response, ["dataValues", "id"])}`, response: responseData }) - return responseData - } + updateDraftDataset = async (draftDataset: Record): Promise> => { - mergeDatasetConfigs = (defaultConfig: Record, requestPayload: Record): Record => { - const { id, dataset_id } = requestPayload; - const datasetId = !id ? dataset_id : id - const modifyPayload = { ...requestPayload, id: datasetId, router_config: { topic: datasetId } } - const defaults = _.cloneDeep(defaultConfig) - const datasetConfigs = _.merge(defaults, modifyPayload) - return datasetConfigs + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }}) + const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key } + logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }) + return responseData; } - getDatasetConnectors = (connectorConfigs: Array>): Array> => { - - if (!_.isEmpty(connectorConfigs)) { - const uniqueConnectors = _.uniqWith(connectorConfigs, (a: Record, b: Record) => { - return _.isEqual(a.connector_id, b.connector_id) && _.isEqual(a.connector_config, b.connector_config) - }) - return _.map(uniqueConnectors, (config) => { - return { - connector_id: config.connector_id, - connector_config: cipherService.encrypt(JSON.stringify(config.connector_config)), - operations_config: config.operations_config - } - }) - } - return [] - } + createDraftDataset = async (draftDataset: Record): Promise> => { - getDatasetTransformations = (configs: Array>): Array> => { - - if (configs) { - return _.uniqBy(configs, "field_key") - } - return [] + const response = await DatasetDraft.create(draftDataset) + const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: draftDataset.version_key } + logger.info({ draftDataset, message: `Dataset Created Successfully with id:${_.get(response, ["dataValues", "id"])}`, response: responseData }) + return responseData } + + } export const datasetService = new DatasetService(); \ No newline at end of file From e78858b7ec0a12e5b41365a7de127f69591fa256 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 4 Jul 2024 22:57:45 +0530 Subject: [PATCH 008/235] #OBS-I115: Dataset read API refactoring --- .../DatasetCreateValidationSchema.json | 2 +- .../v2/controllers/DatasetRead/DatasetRead.ts | 229 +++++------------- .../DatasetUpdateValidationSchema.json | 2 +- api-service/src/v2/models/DatasetDraft.ts | 4 - api-service/src/v2/services/DatasetService.ts | 163 +++++++++++-- 5 files changed, 207 insertions(+), 193 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index b6810c60..55cde742 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -241,7 +241,7 @@ "items": { "type": "object", "properties": { - "denorm_dataset": { + "dataset_id": { "type": "string", "minLength": 1 }, diff --git a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts index 11b4ef01..c6ba7a59 100644 --- a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts @@ -1,198 +1,91 @@ import { Request, Response } from "express"; -import { DatasetStatus } from "../../types/DatasetModels"; +import httpStatus from "http-status"; import _ from "lodash"; -import { validDatasetFields } from "../../configs/DatasetConfigDefault"; +import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { ErrorObject } from "../../types/ResponseModel"; -import logger from "../../logger"; -import httpStatus from "http-status"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { DatasetTransformations } from "../../models/Transformation"; import { DatasetDraft } from "../../models/DatasetDraft"; -import { Dataset } from "../../models/Dataset"; import { datasetService } from "../../services/DatasetService"; -import { Datasource } from "../../models/Datasource"; -import { DatasetSourceConfig } from "../../models/DatasetSourceConfig"; -import { DatasourceDraft } from "../../models/DatasourceDraft"; -import { DatasetSourceConfigDraft } from "../../models/DatasetSourceConfigDraft"; export const apiId = "api.datasets.read"; export const errorCode = "DATASET_READ_FAILURE" -const datasetRead = async (req: Request, res: Response) => { - const { dataset_id } = req.params; - const resmsgid = _.get(res, "resmsgid"); - try { - const { fields, status = DatasetStatus.Live, mode } = req.query; - - setReqDatasetId(req, dataset_id) - - const invalidFields = !_.isEmpty(fields) ? getInvalidFields({ datasetFields: fields, status }) : [] - if (!_.isEmpty(invalidFields)) { - const code = "DATASET_INVALID_FIELDS" - logger.error({ code, apiId, dataset_id, resmsgid, message: `The specified fields [${invalidFields}] in the dataset cannot be found` }) - return ResponseHandler.errorResponse({ - code, - message: `The specified fields [${invalidFields}] in the dataset cannot be found.`, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetModel = getDatasetModel(status); - const fieldValue = !_.isEmpty(fields) ? transformFieldValues({ fields, status }) : "*" - let data: any = {} - if (!_.isEmpty(fieldValue) && mode !== "edit") { - const results = await datasetModel.findAll({ where: { id: dataset_id }, ...(fieldValue !== "*" && { attributes: fieldValue }), raw: true }) - if (_.isEmpty(results)) { - const code = "DATASET_NOT_FOUND" - logger.error({ code, apiId, dataset_id, resmsgid, message: `Dataset with the given dataset_id:${dataset_id} not found` }) - return ResponseHandler.errorResponse({ - code, - message: "Dataset with the given dataset_id not found", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); - } - data = _.first(results) - } - - if (mode === "edit") { - data = await getDatasetByLive(dataset_id) - } +// TODO: Move this to a config +const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config"] - const responseData = await transformResponseData({ status, dataset_id, data, fields }); - logger.info({ apiId, resmsgid, message: `Dataset Read Successfully with id:${dataset_id}`, response: responseData }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); - } catch (error: any) { - const code = _.get(error, "code") || errorCode - logger.error({ ...error, apiId, code, dataset_id, resmsgid }) - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code, message: "Failed to read dataset" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } -} - -const setReqDatasetId = (req: Request, dataset_id: string) => { - if (dataset_id) { - return _.set(req, "dataset_id", dataset_id) - } -} - -const getDatasetModel = (status: string | any) => { - if (status === DatasetStatus.Draft || status === DatasetStatus.ReadyToPublish) { - return DatasetDraft; +const isValidRequest = (req: Request, res: Response): boolean => { + const { dataset_id } = req.params; + const fields = req.query.fields; + if(fields && typeof fields !== 'string') { + logger.error({ code: "DATASET_INVALID_FIELDS_VAL", apiId, dataset_id, message: `The specified fields [${fields}] in the query param is not a string.` }) + ResponseHandler.errorResponse({ + code: "DATASET_INVALID_FIELDS_VAL", + message: `The specified fields [${fields}] in the query param is not a string.`, + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + return false; } - return Dataset; -} - -const getInvalidFields = (payload: Record): Record => { - const { datasetFields, status } = payload - const fieldValues = _.split(datasetFields, ",") - if (!(status == DatasetStatus.Draft || status == DatasetStatus.ReadyToPublish)) { - validDatasetFields.splice(_.indexOf(validDatasetFields, "version_key", 1)) - return _.difference(fieldValues, validDatasetFields) + const fieldValues = _.split(fields, ",") + const invalidFields = _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes())) + if (!_.isEmpty(invalidFields)) { + logger.error({ code: "DATASET_INVALID_FIELDS", apiId, dataset_id, message: `The specified fields [${invalidFields}] in the dataset cannot be found` }) + ResponseHandler.errorResponse({ + code: "DATASET_INVALID_FIELDS", + message: `The specified fields [${invalidFields}] in the dataset cannot be found.`, + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + return false; } - const invalidFields = _.difference(fieldValues, validDatasetFields) - return invalidFields; -} -const transformFieldValues = (datasetFields: Record) => { - const { status, fields } = datasetFields; - const updatedFields = _.remove(_.split(fields, ","), (newField) => newField !== "transformations_config") - if (!(status === DatasetStatus.Draft || status === DatasetStatus.ReadyToPublish) && _.includes(updatedFields, "version")) { - const fieldIndex = _.indexOf(updatedFields, "version") - updatedFields[fieldIndex] = "data_version" - } - return updatedFields + return true; } -const transformResponseData = async (payload: Record) => { - const { data, dataset_id, status, fields } = payload - const transformationConfigExist = _.includes(_.split(fields, ","), "transformations_config") - if (transformationConfigExist || _.isEmpty(fields)) { - const transformationModel = getTransfomationModel(status) - const transformations = await transformationModel.findAll({ where: { dataset_id }, raw: true, attributes: ["field_key", "transformation_function", "mode", "metadata"] }) - _.set(data, "transformations_config", transformations) - } - const liveDatasetVersion = _.get(data, "data_version") - const updatedResponse = liveDatasetVersion ? { ..._.omit(data, ["data_version"]), version: liveDatasetVersion } : data - return updatedResponse -} +const datasetRead = async (req: Request, res: Response) => { -const getTransfomationModel = (status: string) => { - if (status === DatasetStatus.Draft || status === DatasetStatus.ReadyToPublish) { - return DatasetTransformationsDraft + if(!isValidRequest(req, res)) { + return; } - return DatasetTransformations -} - -const getDatasetByLive = async (dataset_id: string) => { - const draftRecord = await DatasetDraft.findOne({ where: { id: dataset_id }, raw: true }) - const liveDataset = await datasetService.getDataset(dataset_id, true) - if (_.isEmpty(liveDataset)) { - throw { + const { dataset_id } = req.params; + const { fields, mode } = req.query; + const attributes = (fields === null) ? defaultFields : _.split(fields, ","); + const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes) + if(!dataset) { + logger.error({ code: "DATASET_NOT_FOUND", apiId, dataset_id, message: `Dataset with the given dataset_id:${dataset_id} not found` }) + ResponseHandler.errorResponse({ code: "DATASET_NOT_FOUND", - message: "Failed to fetch live dataset", + message: "Dataset with the given dataset_id not found", statusCode: 404, errCode: "NOT_FOUND" - } as ErrorObject - } - if (_.isEmpty(draftRecord)) { - const datasetConfig = modifyDatasetConfig(liveDataset) - const [liveTransformation, liveDatasource, liveSourceConfigs] = await Promise.all([getLiveTransformations(dataset_id), getLiveDatasources(dataset_id), getLiveSourceConfigs(dataset_id)]) - const transformationConfig = updateConfigs(liveTransformation) - const datasourceConfig = updateConfigs(liveDatasource) - const sourceConfigs = updateConfigs(liveSourceConfigs) - const response = await DatasetDraft.create(datasetConfig) - await DatasetTransformationsDraft.bulkCreate(transformationConfig) - await DatasourceDraft.bulkCreate(datasourceConfig) - await DatasetSourceConfigDraft.bulkCreate(sourceConfigs) - return _.get(response, "dataValues") - } - if (_.includes([DatasetStatus.Live], _.get(draftRecord, "status"))) { - const [liveTransformation, liveDatasource, liveSourceConfigs] = await Promise.all([getLiveTransformations(dataset_id), getLiveDatasources(dataset_id), getLiveSourceConfigs(dataset_id)]) - await Promise.all([updateLiveConfigs(liveTransformation, DatasetTransformationsDraft), updateLiveConfigs(liveDatasource, DatasourceDraft), updateLiveConfigs(liveSourceConfigs, DatasetSourceConfigDraft)]) - const updatedPayload = modifyDatasetConfig(liveDataset) - await DatasetDraft.update(updatedPayload, { where: { id: _.get(draftRecord, "id") } }) - return updatedPayload + } as ErrorObject, req, res); + } else { + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } - return draftRecord } -const modifyDatasetConfig = (dataset: Record) => { - const updatedPyload = _.omit(dataset, ["data_version", "created_date", "updated_date", "published_date", "status"]) - const version_key = Date.now().toString() - return { ...updatedPyload, version_key, version: _.get(dataset, "data_version"), status: DatasetStatus.Draft, api_version: "v2" } -} - -const updateConfigs = (configs: Record) => { - return _.map(configs, fields => { - const updatedPayload = _.omit(fields, ["created_date", "updated_date", "published_date", "status"]) - return updatedPayload - }) -} - -const getLiveTransformations = async (dataset_id: string) => { - return DatasetTransformations.findAll({ where: { dataset_id }, raw: true }) -} - -const getLiveSourceConfigs = async (dataset_id: string) => { - return DatasetSourceConfig.findAll({ where: { dataset_id }, raw: true }) -} +const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { + + const attrs = _.union(attributes, ["dataset_config", "api_version", "type"]) + const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); + if(draftDataset) { // Contains a draft + const apiVersion = _.get(draftDataset, ["api_version"]); + const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset) + return _.pick(dataset, attributes); + } -const getLiveDatasources = async (dataset_id: string) => { - return Datasource.findAll({ where: { dataset_id }, raw: true }) + const liveDataset = await datasetService.getDataset(datasetId); + if(liveDataset) { + const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) + return _.pick(dataset, attributes); + } + + return null; } -const updateLiveConfigs = async (configs: Record, model: any) => { - return _.map(configs, async fields => { - const updatedPayload = _.omit(fields, ["created_date", "updated_date", "published_date", "status"]) - await model.update({ ...updatedPayload, status: DatasetStatus.Draft }, { where: { id: _.get(updatedPayload, "id") } }) - }) +const readDataset = async (datasetId: string, attributes: string[]): Promise => { + const dataset = await datasetService.getDataset(datasetId, attributes); + return dataset; } export default datasetRead; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 95ef5ee5..630cfbcc 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -268,7 +268,7 @@ "value": { "type": "object", "properties": { - "denorm_dataset": { + "dataset_id": { "type": "string", "minLength": 1 }, diff --git a/api-service/src/v2/models/DatasetDraft.ts b/api-service/src/v2/models/DatasetDraft.ts index b183289e..d049abf5 100644 --- a/api-service/src/v2/models/DatasetDraft.ts +++ b/api-service/src/v2/models/DatasetDraft.ts @@ -90,10 +90,6 @@ export const DatasetDraft = sequelize.define("datasets_draft", { allowNull: false, defaultValue: "SYSTEM", }, - client_state: { - type: DataTypes.JSON, - defaultValue: {} - }, version_key: { type: DataTypes.STRING, allowNull: false diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index c40f07c6..7aa44806 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -1,17 +1,22 @@ import _ from "lodash"; import logger from "../logger"; -import { cipherService } from "./CipherService"; -import { defaultDatasetConfig } from "../configs/DatasetConfigDefault"; import { Dataset } from "../models/Dataset"; import { DatasetDraft } from "../models/DatasetDraft"; import { DatasetTransformations } from "../models/Transformation"; import { DatasetTransformationsDraft } from "../models/TransformationDraft"; - +import Model from "sequelize/types/model"; +import { DatasetSourceConfigDraft } from "../models/DatasetSourceConfigDraft"; +import { sequelize } from "../connections/databaseConnection"; +import { DatasetSourceConfig } from "../models/DatasetSourceConfig"; +import { ConnectorInstances } from "../models/ConnectorInstances"; class DatasetService { - getDataset = async (datasetId: string, raw = false): Promise => { - return Dataset.findOne({ where: { id: datasetId }, raw: raw }); + getDataset = async (datasetId: string, attributes?: string[], raw = false): Promise => { + if(attributes) + return Dataset.findOne({ where: { id: datasetId }, attributes, raw: raw }); + else + return Dataset.findOne({ where: { id: datasetId }, raw: raw }); } getDuplicateDenormKey = (denormConfig: Record): Array => { @@ -34,35 +39,155 @@ class DatasetService { } } - getDraftDataset = async (dataset_id: string) => { - return DatasetDraft.findOne({ where: { dataset_id }, raw: true }); + getDraftDataset = async (dataset_id: string, attributes?: string[]) => { + if(attributes) + return DatasetDraft.findOne({ where: { dataset_id }, attributes, raw: true }); + else + return DatasetDraft.findOne({ where: { dataset_id }, raw: true }); + } + + getDraftTransformations = async (dataset_id: string, attributes?: string[]) => { + if(attributes) + return DatasetTransformationsDraft.findAll({ where: { dataset_id }, attributes, raw: true }); + else + return DatasetTransformationsDraft.findAll({ where: { dataset_id }, raw: true }); + } + + getDraftConnectors = async (dataset_id: string, attributes?: string[]) => { + if(attributes) + return DatasetSourceConfigDraft.findAll({ where: { dataset_id }, attributes, raw: true }); + else + return DatasetSourceConfigDraft.findAll({ where: { dataset_id }, raw: true }); } - getDraftTransformations = async (dataset_id: string) => { - return DatasetTransformationsDraft.findAll({ where: { dataset_id }, raw: true }); + getConnectorsV1 = async (dataset_id: string, attributes?: string[]) => { + if(attributes) + return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); + else + return DatasetSourceConfig.findAll({ where: { dataset_id }, raw: true }); } - getTransformations = async (dataset_id: string) => { - return DatasetTransformations.findAll({ where: { dataset_id }, raw: true }); + getConnectors = async (dataset_id: string, attributes?: string[]) => { + if(attributes) + return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); + else + return ConnectorInstances.findAll({ where: { dataset_id }, raw: true }); + } + + getTransformations = async (dataset_id: string, attributes?: string[]) => { + if(attributes) + return DatasetTransformations.findAll({ where: { dataset_id }, attributes, raw: true }); + else + return DatasetTransformations.findAll({ where: { dataset_id }, raw: true }); } updateDraftDataset = async (draftDataset: Record): Promise> => { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }}) - const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key } - logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }) + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }}); + const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key }; + logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }); return responseData; } createDraftDataset = async (draftDataset: Record): Promise> => { - const response = await DatasetDraft.create(draftDataset) - const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: draftDataset.version_key } - logger.info({ draftDataset, message: `Dataset Created Successfully with id:${_.get(response, ["dataValues", "id"])}`, response: responseData }) - return responseData + const response = await DatasetDraft.create(draftDataset); + const responseData = { id: _.get(response, ["dataValues", "id"]) || "", version_key: draftDataset.version_key }; + logger.info({ draftDataset, message: `Dataset Created Successfully with id:${_.get(response, ["dataValues", "id"])}`, response: responseData }); + return responseData; } - + migrateDraftDataset = async (datasetId: string, dataset: Model): Promise => { + let draftDataset : Record = { + api_version: "v2" + } + const dataset_config:any = _.get(dataset, "dataset_config"); + draftDataset["dataset_config"] = { + indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, + keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, + cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} + } + const transformations = await this.getDraftTransformations(datasetId, ["field_key", "transformation_function", "mode", "metadata"]); + draftDataset["transformations_config"] = _.map(transformations, (config) => { + return { + field_key: _.get(config, ["field_key"]), + transformation_function: _.get(config, ["transformation_function"]), + mode: _.get(config, ["mode"]), + datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", + category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + } + }) + const connectors = await this.getDraftConnectors(datasetId, ["connector_type", "connector_config"]); + draftDataset["connectors_config"] = _.map(connectors, (config) => { + return { + connector_id: _.get(config, ["connector_type"]), + connector_config: _.get(config, ["connector_config"]), + version: "v1" + } + }) + + const transaction = await sequelize.transaction(); + try { + await DatasetDraft.update(draftDataset, { where: { id: datasetId }, transaction}); + await DatasetTransformationsDraft.destroy({ where: { dataset_id: datasetId }, transaction }); + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: datasetId }, transaction }); + } catch(err) { + await transaction.rollback(); + throw err; + } + return await this.getDraftDataset(datasetId); + } + + getTransformationCategory = (section: string):string => { + switch(section) { + case "pii": + return "pii"; + case "additionalFields": + return "derived"; + default: + return "transform"; + } + } + + createDraftDatasetFromLive = async (dataset: Model) => { + let draftDataset:any = _.omit(dataset.toJSON, ["created_date", "updated_date", "published_date"]); + const dataset_config:any = _.get(dataset, "dataset_config"); + const api_version:any = _.get(dataset, "api_version"); + if(api_version === "v1") { + draftDataset["dataset_config"] = { + indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, + keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, + cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} + } + const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["connector_type", "connector_config"]); + draftDataset["connectors_config"] = _.map(connectors, (config) => { + return { + connector_id: _.get(config, ["connector_type"]), + connector_config: _.get(config, ["connector_config"]), + version: "v1" + } + }) + const transformations = await this.getDraftTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); + draftDataset["transformations_config"] = _.map(transformations, (config) => { + return { + field_key: _.get(config, ["field_key"]), + transformation_function: _.get(config, ["transformation_function"]), + mode: _.get(config, ["mode"]), + datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", + category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + } + }) + draftDataset["api_version"] = "v2" + } else { + const connectors = await this.getConnectors(draftDataset.dataset_id, ["connector_id", "connector_config", "operations_config"]); + draftDataset["connectors_config"] = connectors + const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "datatype", "category"]); + draftDataset["transformations_config"] = transformations + } + + await DatasetDraft.create(draftDataset); + return await this.getDraftDataset(draftDataset.dataset_id); + } } From 32974d825a288ef28955c90a5bae7c3cb599a912 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 4 Jul 2024 23:01:45 +0530 Subject: [PATCH 009/235] #OBS-I115: Dataset update API refactoring --- .../v2/controllers/DataExhaust/DataExhaustController.ts | 5 ++--- .../controllers/DataIngestion/DataIngestionController.ts | 5 ++--- .../DatasetStatusTransition/DatasetStatusTransition.ts | 7 +++---- .../src/v2/controllers/EventValidation/EventValidation.ts | 6 +++--- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts b/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts index d8cc88d1..cbc6f52a 100644 --- a/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts +++ b/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts @@ -4,7 +4,7 @@ import httpStatus from "http-status"; import { getDateRange, isValidDateRange } from "../../utils/common"; import { config } from "../../configs/Config"; import moment from "moment"; -import { getDataset, setReqDatasetId } from "../../services/DatasetService"; +import { datasetService } from "../../services/DatasetService"; import * as _ from "lodash"; import logger from "../../logger"; import { cloudProvider } from "../../services/CloudServices"; @@ -14,10 +14,9 @@ export const dataExhaust = async (req: Request, res: Response) => { const { datasetId } = params; const { type }: any = req.query; const momentFormat = "YYYY-MM-DD"; - setReqDatasetId(req, datasetId) const verifyDatasetExists = async (datasetId: string) => { - const dataset = await getDataset(datasetId) + const dataset = await datasetService.getDataset(datasetId) return dataset; } diff --git a/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts index 15f80da4..60f3a3a4 100644 --- a/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts @@ -4,7 +4,7 @@ import validationSchema from "./validationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { send } from "../../connections/kafkaConnection"; -import { getDataset, setReqDatasetId } from "../../services/DatasetService"; +import { datasetService } from "../../services/DatasetService"; import logger from "../../logger"; import { config } from "../../configs/Config"; @@ -29,14 +29,13 @@ const dataIn = async (req: Request, res: Response) => { try { const requestBody = req.body; const datasetId = req.params.datasetId.trim(); - setReqDatasetId(req, datasetId) const isValidSchema = schemaValidation(requestBody, validationSchema) if (!isValidSchema?.isValid) { logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" }) return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_INGESTION_INVALID_INPUT" }, req, res); } - const dataset = await getDataset(datasetId) + const dataset = await datasetService.getDataset(datasetId) if (!dataset) { logger.error({ apiId, message: `Dataset with id ${datasetId} not found in live table`, code: "DATASET_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.datasetNotFound, req, res); diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 96e655cf..4fa47f85 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -2,7 +2,7 @@ import { Request, Response } from "express"; import _ from "lodash"; import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { getDataset, getDraftDataset, setReqDatasetId } from "../../services/DatasetService"; +import { datasetService } from "../../services/DatasetService"; import { ErrorObject } from "../../types/ResponseModel"; import { schemaValidation } from "../../services/ValidationService"; import StatusTransitionSchema from "./RequestValidationSchema.json"; @@ -43,7 +43,6 @@ const datasetStatusTransition = async (req: Request, res: Response) => { const resmsgid = _.get(res, "resmsgid"); try { const { dataset_id, status } = _.get(requestBody, "request"); - setReqDatasetId(req, dataset_id) const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) if (!isRequestValid.isValid) { @@ -107,10 +106,10 @@ const fetchDataset = async (configs: Record) => { return getDraftDatasetRecord(dataset_id) } if (_.includes([DatasetAction.Live], status)) { - return getDraftDataset(dataset_id) + return datasetService.getDraftDataset(dataset_id) } if (_.includes([DatasetAction.Retire], status)) { - return getDataset(dataset_id) + return datasetService.getDataset(dataset_id) } } diff --git a/api-service/src/v2/controllers/EventValidation/EventValidation.ts b/api-service/src/v2/controllers/EventValidation/EventValidation.ts index 718a4785..bd850968 100644 --- a/api-service/src/v2/controllers/EventValidation/EventValidation.ts +++ b/api-service/src/v2/controllers/EventValidation/EventValidation.ts @@ -4,7 +4,7 @@ import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import validationSchema from "./RequestValidationSchema.json" import logger from "../../logger"; -import { getDataset, getDraftDataset } from "../../services/DatasetService"; +import { datasetService } from "../../services/DatasetService"; const apiId = "api.schema.validator"; export const eventValidation = async (req: Request, res: Response) => { @@ -25,12 +25,12 @@ export const eventValidation = async (req: Request, res: Response) => { } if (isLive) { - dataset = await getDataset(datasetId, true); + dataset = await datasetService.getDataset(datasetId, undefined, true); schema = _.get(dataset, "data_schema") } if (!isLive) { - dataset = await getDraftDataset(datasetId); + dataset = await datasetService.getDraftDataset(datasetId); schema = _.get(dataset, "data_schema") } From 364f9b8563d84b0eb7969a677a39a4f10a204bf4 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Fri, 5 Jul 2024 11:34:04 +0530 Subject: [PATCH 010/235] #OBS-I115: Dataset list API refactoring --- .../DataExhaust/DataExhaustController.ts | 2 +- .../DataIngestion/DataIngestionController.ts | 2 +- .../v2/controllers/DatasetList/DatasetList.ts | 154 ++++-------------- .../DatasetListValidationSchema.json | 31 +--- .../v2/controllers/DatasetRead/DatasetRead.ts | 4 +- api-service/src/v2/services/DatasetService.ts | 60 +++---- 6 files changed, 62 insertions(+), 191 deletions(-) diff --git a/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts b/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts index cbc6f52a..ceeb3358 100644 --- a/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts +++ b/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts @@ -16,7 +16,7 @@ export const dataExhaust = async (req: Request, res: Response) => { const momentFormat = "YYYY-MM-DD"; const verifyDatasetExists = async (datasetId: string) => { - const dataset = await datasetService.getDataset(datasetId) + const dataset = await datasetService.getDataset(datasetId, ["id"], true) return dataset; } diff --git a/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts index 60f3a3a4..49e1ebf6 100644 --- a/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts @@ -35,7 +35,7 @@ const dataIn = async (req: Request, res: Response) => { logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" }) return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_INGESTION_INVALID_INPUT" }, req, res); } - const dataset = await datasetService.getDataset(datasetId) + const dataset = await datasetService.getDataset(datasetId, ["id"], true) if (!dataset) { logger.error({ apiId, message: `Dataset with id ${datasetId} not found in live table`, code: "DATASET_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.datasetNotFound, req, res); diff --git a/api-service/src/v2/controllers/DatasetList/DatasetList.ts b/api-service/src/v2/controllers/DatasetList/DatasetList.ts index 81d1a291..3003fc33 100644 --- a/api-service/src/v2/controllers/DatasetList/DatasetList.ts +++ b/api-service/src/v2/controllers/DatasetList/DatasetList.ts @@ -1,146 +1,54 @@ +import _ from "lodash"; +import httpStatus from "http-status"; import { Request, Response } from "express"; +import logger from "../../logger"; import { schemaValidation } from "../../services/ValidationService"; import DatasetCreate from "./DatasetListValidationSchema.json"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { ErrorObject } from "../../types/ResponseModel"; -import { DatasetDraft } from "../../models/DatasetDraft"; -import logger from "../../logger"; -import _ from "lodash"; -import { Dataset } from "../../models/Dataset"; -import httpStatus from "http-status"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { DatasetTransformations } from "../../models/Transformation"; -import { DatasetStatus } from "../../types/DatasetModels"; +import { datasetService } from "../../services/DatasetService"; export const apiId = "api.datasets.list" export const errorCode = "DATASET_LIST_FAILURE" -const liveDatasetStatus = ["Live", "Retired"] +const liveDatasetStatus = ["Live", "Retired", "Purged"] const draftDatasetStatus = ["Draft", "ReadyToPublish"] +const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config"] const datasetList = async (req: Request, res: Response) => { + const requestBody = req.body; const msgid = _.get(req, ["body", "params", "msgid"]); const resmsgid = _.get(res, "resmsgid"); - try { - const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) - if (!isRequestValid.isValid) { - const code = "DATASET_LIST_INPUT_INVALID" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - - const datasetBody = req.body.request; - const datasetList = await getDatasetList(datasetBody) - const responseData = { data: datasetList, count: _.size(datasetList) } - logger.info({ apiId, msgid, requestBody, resmsgid, message: `Datasets are listed successfully with a dataset count (${_.size(datasetList)})` }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); - } catch (error: any) { - logger.error({ ...error, apiId, code: errorCode, msgid, requestBody, resmsgid }); - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: errorCode, message: "Failed to list dataset" } - } - ResponseHandler.errorResponse(errorMessage, req, res); + const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) + if (!isRequestValid.isValid) { + const code = "DATASET_LIST_INPUT_INVALID" + logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) + return ResponseHandler.errorResponse({ + code, + message: isRequestValid.message, + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); } -} -const getDatasetList = async (request: Record): Promise> => { - const { filters = {}, sortBy = [] } = request || {}; - const datasets = await getAllDatasets(filters) - const sortedDatasets: any = getSortedDatasets(datasets, sortBy) - const datasetsList = await transformDatasetList(sortedDatasets) - return datasetsList; + const datasetBody = req.body.request; + const datasetList = await listDatasets(datasetBody) + const responseData = { data: datasetList, count: _.size(datasetList) } + logger.info({ apiId, msgid, requestBody, resmsgid, message: `Datasets are listed successfully with a dataset count (${_.size(datasetList)})` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); + } -const getAllDatasets = async (filters: Record): Promise> => { - const datasetStatus = _.get(filters, "status"); - const { liveDatasetList, draftDatasetList } = await fetchDatasets({ datasetStatus, filters }) - return _.compact(_.concat(liveDatasetList, draftDatasetList)) -} +const listDatasets = async (request: Record): Promise> => { -const fetchDatasets = async (data: Record) => { - const { filters, datasetStatus } = data - if (_.isEmpty(datasetStatus)) { - const [liveDatasetList, draftDatasetList] = await Promise.all([getLiveDatasets(filters, liveDatasetStatus), getDraftDatasets(filters, draftDatasetStatus)]) - return { liveDatasetList, draftDatasetList } - } - let liveDatasetList, draftDatasetList; + const { filters = {} } = request || {}; + const datasetStatus = _.get(filters, "status"); const status = _.isArray(datasetStatus) ? datasetStatus : [datasetStatus] - const draftStatus = _.intersection(status, draftDatasetStatus); - const liveStatus = _.intersection(status, liveDatasetStatus); - if (_.size(liveStatus) > 0) { - liveDatasetList = await getLiveDatasets(filters, liveStatus) - } - if (_.size(draftStatus) > 0) { - draftDatasetList = await getDraftDatasets(filters, draftStatus) - } - return { liveDatasetList, draftDatasetList } -} - -const getSortedDatasets = (datasets: Record, sortOrder: Record): Record => { - if (!_.isEmpty(sortOrder)) { - const fieldValues = _.map(sortOrder, field => _.get(field, "field")) - const orderValues = _.map(sortOrder, field => _.get(field, "order")) - return _.orderBy(datasets, fieldValues, orderValues) - } - return datasets -} - -const transformDatasetList = async (datasets: Record) => { - const { liveDatasetId, draftDatasetId } = getDatasetId(datasets) - const [draftTransformations, liveTransformations] = await Promise.all([getDraftTransformations(draftDatasetId), getLiveTransformations(liveDatasetId)]) - const transformationList = _.concat(liveTransformations, draftTransformations) - const datasetList = _.map(datasets, dataset => { - const transformationConfig = _.compact(_.flatten(_.map(transformationList, (transformations: any) => { - const datasetId = _.get(dataset, "id") - const transformationId = _.get(transformations, "dataset_id") - if (datasetId === transformationId) { - return _.omit(transformations, ["dataset_id"]); - } - }))) - const liveDatasetVersion = _.get(dataset, "data_version") - const updatedList = liveDatasetVersion ? { ..._.omit(dataset, ["data_version"]), version: liveDatasetVersion } : dataset - return { ...updatedList, ...(!_.isEmpty(transformationConfig) && { "transformations_config": transformationConfig }) } - }) - return datasetList -} - -const getDatasetId = (datasets: Record) => { - const liveDatasets = _.filter(datasets, field => { - const { status } = field || {} - return status === DatasetStatus.Live || status === DatasetStatus.Retired - }) - const draftDatasets = _.filter(datasets, field => { - const { status } = field || {} - return status === DatasetStatus.Draft || status === DatasetStatus.ReadyToPublish - }) - const liveDatasetId = _.map(liveDatasets, list => _.get(list, "id")) - const draftDatasetId = _.map(draftDatasets, list => _.get(list, "id")) - return { liveDatasetId, draftDatasetId } -} - -const getDraftDatasets = async (filters: Record, datasetStatus: Array): Promise> => { - return DatasetDraft.findAll({ where: { ...filters, ...(!_.isEmpty(datasetStatus) && { status: datasetStatus }) }, raw: true }); -} - -const getLiveDatasets = async (filters: Record, datasetStatus: Array): Promise> => { - return Dataset.findAll({ where: { ...filters, ...(!_.isEmpty(datasetStatus) && { status: datasetStatus }) }, raw: true }); -} - -const datasetTransformationAttributes = ["dataset_id", "field_key", "transformation_function", "mode", "metadata"] - -const getDraftTransformations = async (dataset_id: Array) => { - return DatasetTransformationsDraft.findAll({ where: { status: draftDatasetStatus, dataset_id }, attributes: datasetTransformationAttributes, raw: true }) -} - -const getLiveTransformations = async (dataset_id: Array) => { - return DatasetTransformations.findAll({ where: { status: liveDatasetStatus, dataset_id }, attributes: datasetTransformationAttributes, raw: true }) + const draftFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? draftDatasetStatus : _.intersection(datasetStatus, draftDatasetStatus)); + const liveFilters = _.set(_.cloneDeep(filters), "status", _.isEmpty(status) ? liveDatasetStatus : _.intersection(datasetStatus, liveDatasetStatus)); + const liveDatasetList = await datasetService.findDatasets(liveFilters, defaultFields, [["updated_date", "DESC"]]); + const draftDatasetList = await datasetService.findDraftDatasets(draftFilters, defaultFields, [["updated_date", "DESC"]]); + return _.compact(_.concat(liveDatasetList, draftDatasetList)); } export default datasetList; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetList/DatasetListValidationSchema.json b/api-service/src/v2/controllers/DatasetList/DatasetListValidationSchema.json index f6a6955e..3435d631 100644 --- a/api-service/src/v2/controllers/DatasetList/DatasetListValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetList/DatasetListValidationSchema.json @@ -33,7 +33,7 @@ "type": "array", "items": { "type": "string", - "enum": ["Draft", "Live", "ReadyToPublish", "Retired"] + "enum": ["Draft", "Live", "ReadyToPublish", "Retired", "Purged"] } }, { @@ -44,37 +44,10 @@ }, "type": { "type": "string", - "enum": ["dataset", "master-dataset"] - }, - "dataset_id": { - "type": "string", - "minLength": 1 + "enum": ["event", "transaction", "master"] } }, "additionalProperties": false - }, - "sortBy": { - "type": "array", - "items": { - "type": "object", - "properties": { - "field": { - "type": "string", - "enum": [ - "created_date", - "updated_date", - "published_date", - "name" - ] - }, - "order": { - "type": "string", - "enum": ["asc", "desc"] - } - }, - "additionalProperties": false, - "required": ["field", "order"] - } } }, "additionalProperties": false diff --git a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts index c6ba7a59..44b20c73 100644 --- a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts @@ -74,7 +74,7 @@ const readDraftDataset = async (datasetId: string, attributes: string[]): Promis return _.pick(dataset, attributes); } - const liveDataset = await datasetService.getDataset(datasetId); + const liveDataset = await datasetService.getDataset(datasetId, undefined, true); if(liveDataset) { const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) return _.pick(dataset, attributes); @@ -84,7 +84,7 @@ const readDraftDataset = async (datasetId: string, attributes: string[]): Promis } const readDataset = async (datasetId: string, attributes: string[]): Promise => { - const dataset = await datasetService.getDataset(datasetId, attributes); + const dataset = await datasetService.getDataset(datasetId, attributes, true); return dataset; } diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 7aa44806..9d8a613c 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -13,10 +13,11 @@ import { ConnectorInstances } from "../models/ConnectorInstances"; class DatasetService { getDataset = async (datasetId: string, attributes?: string[], raw = false): Promise => { - if(attributes) - return Dataset.findOne({ where: { id: datasetId }, attributes, raw: raw }); - else - return Dataset.findOne({ where: { id: datasetId }, raw: raw }); + return Dataset.findOne({ where: { id: datasetId }, attributes, raw: raw }); + } + + findDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { + return Dataset.findAll({where, attributes, order, raw: true}) } getDuplicateDenormKey = (denormConfig: Record): Array => { @@ -40,45 +41,31 @@ class DatasetService { } getDraftDataset = async (dataset_id: string, attributes?: string[]) => { - if(attributes) - return DatasetDraft.findOne({ where: { dataset_id }, attributes, raw: true }); - else - return DatasetDraft.findOne({ where: { dataset_id }, raw: true }); + return DatasetDraft.findOne({ where: { dataset_id }, attributes, raw: true }); + } + + findDraftDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { + return DatasetDraft.findAll({where, attributes, order, raw: true}) } getDraftTransformations = async (dataset_id: string, attributes?: string[]) => { - if(attributes) - return DatasetTransformationsDraft.findAll({ where: { dataset_id }, attributes, raw: true }); - else - return DatasetTransformationsDraft.findAll({ where: { dataset_id }, raw: true }); + return DatasetTransformationsDraft.findAll({ where: { dataset_id }, attributes, raw: true }); } getDraftConnectors = async (dataset_id: string, attributes?: string[]) => { - if(attributes) - return DatasetSourceConfigDraft.findAll({ where: { dataset_id }, attributes, raw: true }); - else - return DatasetSourceConfigDraft.findAll({ where: { dataset_id }, raw: true }); + return DatasetSourceConfigDraft.findAll({ where: { dataset_id }, attributes, raw: true }); } getConnectorsV1 = async (dataset_id: string, attributes?: string[]) => { - if(attributes) - return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); - else - return DatasetSourceConfig.findAll({ where: { dataset_id }, raw: true }); + return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); } getConnectors = async (dataset_id: string, attributes?: string[]) => { - if(attributes) - return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); - else - return ConnectorInstances.findAll({ where: { dataset_id }, raw: true }); + return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); } getTransformations = async (dataset_id: string, attributes?: string[]) => { - if(attributes) - return DatasetTransformations.findAll({ where: { dataset_id }, attributes, raw: true }); - else - return DatasetTransformations.findAll({ where: { dataset_id }, raw: true }); + return DatasetTransformations.findAll({ where: { dataset_id }, attributes, raw: true }); } updateDraftDataset = async (draftDataset: Record): Promise> => { @@ -98,6 +85,7 @@ class DatasetService { } migrateDraftDataset = async (datasetId: string, dataset: Model): Promise => { + let draftDataset : Record = { api_version: "v2" } @@ -139,6 +127,7 @@ class DatasetService { } getTransformationCategory = (section: string):string => { + switch(section) { case "pii": return "pii"; @@ -150,6 +139,7 @@ class DatasetService { } createDraftDatasetFromLive = async (dataset: Model) => { + let draftDataset:any = _.omit(dataset.toJSON, ["created_date", "updated_date", "published_date"]); const dataset_config:any = _.get(dataset, "dataset_config"); const api_version:any = _.get(dataset, "api_version"); @@ -162,18 +152,18 @@ class DatasetService { const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { return { - connector_id: _.get(config, ["connector_type"]), - connector_config: _.get(config, ["connector_config"]), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), version: "v1" } }) const transformations = await this.getDraftTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); draftDataset["transformations_config"] = _.map(transformations, (config) => { return { - field_key: _.get(config, ["field_key"]), - transformation_function: _.get(config, ["transformation_function"]), - mode: _.get(config, ["mode"]), - datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", + field_key: _.get(config, "field_key"), + transformation_function: _.get(config, "transformation_function"), + mode: _.get(config, "mode"), + datatype: _.get(config, "metadata._transformedFieldDataType") || "string", category: this.getTransformationCategory(_.get(config, ["metadata.section"])) } }) @@ -184,7 +174,7 @@ class DatasetService { const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "datatype", "category"]); draftDataset["transformations_config"] = transformations } - + draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version await DatasetDraft.create(draftDataset); return await this.getDraftDataset(draftDataset.dataset_id); } From 86ccc30aa8a4fb48eebd8c9e0450cb156db543b3 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Fri, 5 Jul 2024 13:31:30 +0530 Subject: [PATCH 011/235] #0000: adding command api --- .gitignore | 17 +- command-service/Dockerfile | 17 +- command-service/README.md | 9 +- command-service/docs/pii_swagger_doc.yml | 147 +++++++ .../connector-cron-jobs/Chart.yaml | 24 ++ .../templates/_helpers.tpl | 74 ++++ .../templates/cronjob.yaml | 59 +++ .../connector-cron-jobs/templates/rbac.yaml | 48 +++ .../connector-cron-jobs/values.yaml | 29 ++ .../spark-connector-cron/Chart.lock | 6 + .../spark-connector-cron/Chart.yaml | 11 + .../charts/common/.helmignore | 23 ++ .../charts/common/Chart.yaml | 12 + .../charts/common/templates/_affinities.tpl | 139 +++++++ .../charts/common/templates/_capabilities.tpl | 229 +++++++++++ .../charts/common/templates/_configs.tpl | 0 .../charts/common/templates/_errors.tpl | 28 ++ .../charts/common/templates/_images.tpl | 117 ++++++ .../charts/common/templates/_ingress.tpl | 73 ++++ .../charts/common/templates/_labels.tpl | 46 +++ .../charts/common/templates/_names.tpl | 71 ++++ .../charts/common/templates/_secrets.tpl | 172 ++++++++ .../charts/common/templates/_storage.tpl | 28 ++ .../charts/common/templates/_tplvalues.tpl | 38 ++ .../charts/common/templates/_utils.tpl | 77 ++++ .../charts/common/templates/_variables.tpl | 41 ++ .../charts/common/templates/_warnings.tpl | 19 + .../templates/validations/_cassandra.tpl | 77 ++++ .../common/templates/validations/_mariadb.tpl | 108 ++++++ .../common/templates/validations/_mongodb.tpl | 113 ++++++ .../common/templates/validations/_mysql.tpl | 108 ++++++ .../templates/validations/_postgresql.tpl | 134 +++++++ .../common/templates/validations/_redis.tpl | 81 ++++ .../templates/validations/_validations.tpl | 51 +++ .../charts/common/values.yaml | 82 ++++ .../spark-connector-cron/templates/NOTES.txt | 0 .../templates/_base_serviceAccount.tpl | 4 + .../templates/_cron_release_name.tpl | 4 + .../spark-connector-cron/templates/_image.tpl | 10 + .../templates/_namespace.tpl | 14 + .../templates/cronjob.yaml | 82 ++++ .../spark-connector-cron/templates/rbac.yaml | 48 +++ .../templates/serviceaccount.yaml | 11 + .../spark-connector-cron/values.yaml | 166 ++++++++ command-service/requirements.txt | 16 +- command-service/src/command/__init__.py | 6 +- .../src/command/alert_manager_command.py | 213 ++++++++++ .../src/command/command_executor.py | 104 ++++- .../src/command/connector_command.py | 168 ++++++++ .../src/command/connector_registry.py | 367 ++++++++++++++++++ .../src/command/dataset_command.py | 95 +++++ command-service/src/command/db_command.py | 263 +++++++++++++ command-service/src/command/druid_command.py | 52 +++ command-service/src/command/flink_command.py | 60 ++- command-service/src/command/icommand.py | 2 +- .../src/command/telemetry_command.py | 59 +++ command-service/src/config/__init__.py | 3 +- command-service/src/config/config.py | 19 +- command-service/src/config/pii_rules.yml | 118 ++++++ command-service/src/config/service_config.yml | 178 ++++++++- command-service/src/exception/exception.py | 10 +- command-service/src/metrics/__init__.py | 3 + command-service/src/metrics/helper.py | 38 ++ command-service/src/metrics/metrics.py | 41 ++ command-service/src/model/__init__.py | 11 +- command-service/src/model/data_models.py | 109 +++++- command-service/src/model/db_models.py | 127 ++++++ command-service/src/model/telemetry_models.py | 75 ++++ command-service/src/routes.py | 265 ++++++++++++- command-service/src/service/__init__.py | 4 +- command-service/src/service/backup_service.py | 152 ++++++++ command-service/src/service/db_service.py | 67 ++++ .../src/service/detect_pii_service.py | 33 ++ command-service/src/service/http_service.py | 39 +- .../src/service/prometheus_backup.py | 48 +++ command-service/src/service/re_pii_model.py | 64 +++ .../src/service/telemetry_service.py | 38 ++ command-service/src/stubs/snippets.py | 13 + 78 files changed, 5408 insertions(+), 99 deletions(-) create mode 100644 command-service/docs/pii_swagger_doc.yml create mode 100644 command-service/helm-charts/connector-cron-jobs/Chart.yaml create mode 100644 command-service/helm-charts/connector-cron-jobs/templates/_helpers.tpl create mode 100644 command-service/helm-charts/connector-cron-jobs/templates/cronjob.yaml create mode 100644 command-service/helm-charts/connector-cron-jobs/templates/rbac.yaml create mode 100644 command-service/helm-charts/connector-cron-jobs/values.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/Chart.lock create mode 100644 command-service/helm-charts/spark-connector-cron/Chart.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/.helmignore create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/Chart.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_affinities.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_capabilities.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_configs.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_errors.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_images.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_ingress.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_labels.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_names.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_secrets.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_storage.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_tplvalues.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_utils.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_variables.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/_warnings.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_cassandra.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mariadb.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mongodb.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mysql.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_postgresql.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_redis.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_validations.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/charts/common/values.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/templates/NOTES.txt create mode 100644 command-service/helm-charts/spark-connector-cron/templates/_base_serviceAccount.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/templates/_image.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/templates/_namespace.tpl create mode 100644 command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/templates/rbac.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml create mode 100644 command-service/helm-charts/spark-connector-cron/values.yaml create mode 100644 command-service/src/command/alert_manager_command.py create mode 100644 command-service/src/command/connector_command.py create mode 100644 command-service/src/command/connector_registry.py create mode 100644 command-service/src/command/dataset_command.py create mode 100644 command-service/src/command/db_command.py create mode 100644 command-service/src/command/druid_command.py create mode 100644 command-service/src/command/telemetry_command.py create mode 100644 command-service/src/config/pii_rules.yml create mode 100644 command-service/src/metrics/__init__.py create mode 100644 command-service/src/metrics/helper.py create mode 100644 command-service/src/metrics/metrics.py create mode 100644 command-service/src/model/db_models.py create mode 100644 command-service/src/model/telemetry_models.py create mode 100644 command-service/src/service/backup_service.py create mode 100644 command-service/src/service/db_service.py create mode 100644 command-service/src/service/detect_pii_service.py create mode 100644 command-service/src/service/prometheus_backup.py create mode 100644 command-service/src/service/re_pii_model.py create mode 100644 command-service/src/service/telemetry_service.py create mode 100644 command-service/src/stubs/snippets.py diff --git a/.gitignore b/.gitignore index aafda3a6..4e1e06fe 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,15 @@ -.vscode -.idea +coverage +node_modules +.nyc_output +dist +credentials.js +uploads/ +.env +# Byte-compiled / optimized / DLL files +*.pyc +__pycache__/ +.vscode/ +*.DS_Store +#to ignore embedded postgres data files +data +connector-registry \ No newline at end of file diff --git a/command-service/Dockerfile b/command-service/Dockerfile index 4d0d51f3..2aa9a1b5 100644 --- a/command-service/Dockerfile +++ b/command-service/Dockerfile @@ -1,11 +1,22 @@ -FROM --platform=linux/amd64 python:3.10-alpine +FROM --platform=linux/amd64 python:3.12-alpine COPY --from=ubuntu /usr/local/bin /usr/local/bin + RUN apk update && apk add curl jq && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ -RUN apk add libcrypto3=3.1.4-r5 -RUN apk upgrade + +RUN cd /tmp && curl -OL https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz \ + && tar -zxvf helm-v3.13.2-linux-amd64.tar.gz \ + && mv linux-amd64/helm /usr/local/bin/helm + +# RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ +# && mv kubectl /usr/local/bin/kubectl && chmod +x /usr/local/bin/kubectl + +# RUN apk add openssl==3.3.0-r2 libcrypto3==3.3.0-r2 || \ +RUN apk add openssl libcrypto3 +RUN apk upgrade --no-cache WORKDIR /app COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt COPY src ./src +COPY helm-charts ./helm-charts WORKDIR /app/src CMD [ "uvicorn", "routes:app", "--host", "0.0.0.0" ] \ No newline at end of file diff --git a/command-service/README.md b/command-service/README.md index 5c06533a..d11413b7 100644 --- a/command-service/README.md +++ b/command-service/README.md @@ -3,12 +3,13 @@ ### Commands * Each command implementation under the command module will extend from icommand interface and will have to implement the execute function. -* The execute function will take a command payload json object and also an action as input. -* Currently implemented FlinkCommand class. +* The execute function will take a command payload json object and also an action as input. For e.g. The DruidCommand class' execute function will take the command payload and SUBMIT_INGESTION_TASK action as two parameters. +* Currently implemented command classes are DruidCommand, FlinkCommand and DbCommand classes. ### Services -Currently, there are one generic service under the services module: +Currently, there are two generic services under the services module: +* DatabaseService implements all the required database operations such as select_one, select_all and upsert operations from Postgresql. The service uses psycopg2 library to connect to Postgres. * HttpService implements GET, POST and DELETE operations. The service uses urllib3 library to invoke http urls. ### Configuration @@ -16,7 +17,7 @@ Currently, there are one generic service under the services module: * The config.py class has an utility implementation `find` to read nested configurations from a yaml file. * The service_config.yml class has all the required configurations for the service. - The flink.jobs configuration is required to specify the list of jobs and the corresponding job_manager_urls. This is required for restarting the required jobs. - - The commands entry will have the workflow of sub-commands for each higher level comamnd. For e.g., RESTART_PIPELINE command is comprised for RESTART_PIPELINE_JOBS sub-command. + - The commands entry will have the workflow of sub-commands for each higher level comamnd. For e.g., PUBLISH_DATASET command is comprised for five sub-commands such as MAKE_DATASET_LIVE, SUBMIT_INGESTION_TASKS, STOP_PIPELINE_JOBS and START_PIPELINE_JOBS. ### Deployment diff --git a/command-service/docs/pii_swagger_doc.yml b/command-service/docs/pii_swagger_doc.yml new file mode 100644 index 00000000..f0992894 --- /dev/null +++ b/command-service/docs/pii_swagger_doc.yml @@ -0,0 +1,147 @@ +openapi: 3.0.0 +info: + title: PII Detection API + description: A simple API for detecting PII fields in dataset sample. + version: 1.0.0 +servers: + - url: http://{url}:{port}/data/v1 +paths: + /analyze/pii: + post: + summary: return fields with PII data flagged with explanation and confidence + operationId: detectPII + requestBody: + description: dataset id and sample single flattened event with all fields populated + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/PIIRequest' + responses: + '200': + description: A list of potential fields with PII + content: + application/json: + schema: + $ref: '#/components/schemas/PIIResponse' + '500': + description: Internal server error + content: + application/json: + schema: + $ref: '#/components/schemas/PIIFailedResponse' +components: + schemas: + PIIRequest: + type: object + properties: + dataset_id: + type: string + example: telemetry-data + data: + type: array + items: + type: object + required: + - dataset_id + - data + PIIResponse: + type: object + properties: + id: + type: string + example: dc235b34-32b1-4d5e-b1c8-985f5e4d30d9 + response_code: + type: string + example: OK + status_code: + type: integer + example: 200 + result: + type: array + items: + type: object + properties: + field: + type: string + description: the flattened column name/path + example: user.mobile + type: + type: string + description: whether the field is detected as a name/phone/email/ID/location + example: phone + score: + type: number + description: a 0-100% value signifying the confidence in the detection + example: 0.75 + reason: + type: array + items: + properties: + code: + type: string + description: reason code for i18n code + example: <> + resourceKey:: + type: string + description: resource bundle id for UI to display proper label + example: pii.descriptions.m001 + region: + type: string + description: Optional. whether related to any specific region + example: IN + score: + type: number + description: Confidence score specific to region + example: 0.75 + required: + - code + - message + ts: + type: string + example: 1705645119000 + params: + type: object + properties: + status: + type: string + required: + - id + - response_code + - status_code + PIIFailedResponse: + type: object + properties: + id: + type: string + example: dc235b34-32b1-4d5e-b1c8-985f5e4d30d9 + response_code: + type: string + example: INTERNAL_SERVER_ERROR + status_code: + type: integer + example: 500 + result: + type: object + properties: + errorCode: + type: integer + example: 500 + errorMsg: + type: string + example: Error parsing the sample event + errorTrace: + type: string + example: "SyntaxError: JSON.parse: bad character in string literal" + ts: + type: string + example: 1705645119000 + params: + type: object + properties: + status: + type: string + required: + - id + - response_code + - status_code diff --git a/command-service/helm-charts/connector-cron-jobs/Chart.yaml b/command-service/helm-charts/connector-cron-jobs/Chart.yaml new file mode 100644 index 00000000..81eb2cf0 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: connector-cron-jobs +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/command-service/helm-charts/connector-cron-jobs/templates/_helpers.tpl b/command-service/helm-charts/connector-cron-jobs/templates/_helpers.tpl new file mode 100644 index 00000000..84c16281 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/templates/_helpers.tpl @@ -0,0 +1,74 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "connector-cron-jobs.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "connector-cron-jobs.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "connector-cron-jobs.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "connector-cron-jobs.labels" -}} +helm.sh/chart: {{ include "connector-cron-jobs.chart" . }} +{{ include "connector-cron-jobs.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "connector-cron-jobs.selectorLabels" -}} +app.kubernetes.io/name: {{ include "connector-cron-jobs.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +# .Values.namespace will get overridden by .Values.global.namespace.chart-name +{{- define "base.namespace" -}} + {{- $chartName := .Chart.Name }} + {{- $namespace := default .Release.Namespace .Values.namespace }} + {{- if .Values.global }} + {{- with .Values.global.namespace }} + {{- if hasKey . $chartName }} + {{- $namespace = index . $chartName }} + {{- end }} + {{- end }} + {{- end }} + {{- $namespace | trunc 63 | trimSuffix "-" }} +{{- end }} + + +{{/* +Create the name of the service account to use +*/}} +{{- define "base.serviceaccountname" -}} + {{- $name := printf "%s-%s" .Chart.Name "sa" }} + {{- default $name .Values.serviceAccount.name }} +{{- end }} diff --git a/command-service/helm-charts/connector-cron-jobs/templates/cronjob.yaml b/command-service/helm-charts/connector-cron-jobs/templates/cronjob.yaml new file mode 100644 index 00000000..a0611094 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/templates/cronjob.yaml @@ -0,0 +1,59 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: "{{ .Values.instance-id }}-cronjob" +spec: + schedule: "{{ .Values.schedule }}" + failedJobsHistoryLimit: 1 + successfulJobsHistoryLimit: 1 + concurrencyPolicy: Forbid + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: OnFailure + containers: + - name: spark-submit-container + image: bitnami/kubectl:latest + pullPolicy: IfNotPresent + command: + {{- if eq .Values.technology "scala" }} + - /bin/sh + - -c + - | + # Wait for the Spark pod to be ready + SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --jars /data/connectors/{{ .Values.connector-source }}/libs/\*.jar --class {{ .Values.main_class }} /data/connectors/{{ .Values.connector-source }}/{{ .Values.main_file }} -c {{ .Values.instance-id }}" + {{- else if eq .Values.technology "python" }} + - /bin/sh + - -c + - | + # Wait for the Spark pod to be ready + SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector-source }}/libs/\* /data/connectors/{{ .Values.connector-source }}/{{ .Values.main_file }} -c {{ .Values.instance-id }}" + {{- end }} + + # - name: connector-cron-jobs + # image: alpine/curl + # command: ["/bin/sh", "-c"] + # args: + # - > + # CURRENT_TIME=$(date +%Y-%m-%dT%H:%M:%S.%sZ) && curl -vi --location + # "{{ .Values.spark.host }}" + # --header "Content-Type: application/json" + # --data + # '{ + # "file": "{{ .Values.file.path }}", + # "args": ["{{ .Values.args }}"], + # "name": "{{ .Values.job.name }}-'"$CURRENT_TIME"'", + # "className": "{{ .Values.class.name }}", + # "executorCores": 1, + # "executorMemory": "1G", + # "numExecutors": 1, + # "conf": { + # "spark.driver.extraJavaOptions": "{{ .Values.spark.driver.extraJavaOptions }}", + # "spark.executor.extraJavaOptions": "{{ .Values.spark.executor.extraJavaOptions }}", + # "spark.master": "spark://{{ .Values.spark.master.host }}:{{ .Values.spark.master.port }}" + # } + # }' diff --git a/command-service/helm-charts/connector-cron-jobs/templates/rbac.yaml b/command-service/helm-charts/connector-cron-jobs/templates/rbac.yaml new file mode 100644 index 00000000..7e291fd6 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/templates/rbac.yaml @@ -0,0 +1,48 @@ +{{- if .Values.rbac.enabled -}} +--- +{{- if .Values.rbac.useClusterRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "common.names.fullname" . }} +rules: +{{- toYaml .Values.rbac.rules | nindent 2 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "common.names.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "base.serviceaccountname" . }} + namespace: {{ include "base.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "common.names.fullname" . }} +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} +rules: +{{- toYaml .Values.rbac.rules | nindent 2 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} +subjects: +- kind: ServiceAccount + name: {{ include "base.serviceaccountname" . }} + namespace: {{ include "base.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "common.names.fullname" . }} +{{- end }} + +{{- end }} + diff --git a/command-service/helm-charts/connector-cron-jobs/values.yaml b/command-service/helm-charts/connector-cron-jobs/values.yaml new file mode 100644 index 00000000..15127206 --- /dev/null +++ b/command-service/helm-charts/connector-cron-jobs/values.yaml @@ -0,0 +1,29 @@ +## JDBC Connector +technology: scala +connector-source: jdbc-connector-1.0.0 +instance-id: nyt-psql.1 +main_class: org.sunbird.obsrv.connector.JDBCConnector +main_file: jdbc-connector-1.0.0.jar + + +## Object Store Connector +# technology: python +# connector-source: object_store_connector-0.1.0 +# instance-id: s3.new-york-taxi-data.1 +# main_file: object_store_connector/__main__.py + +schedule: 0 * * * * # Every hour + +## Defaults + +python_path: /opt/bitnami/python/bin/python + +spark: + host: http://spark-master-svc.spark.svc.cluster.local:8998/batches/ + master: + host: spark-master-svc.spark.svc.cluster.local + port: 7077 + driver: + extraJavaOptions: -Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties + executor: + extraJavaOptions: -Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties diff --git a/command-service/helm-charts/spark-connector-cron/Chart.lock b/command-service/helm-charts/spark-connector-cron/Chart.lock new file mode 100644 index 00000000..13768315 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://nimbushubin.github.io/helmcharts + version: 0.1.0 +digest: sha256:1caa4d36f25ec305383122b0a8a6d585921a608b03d9aaf15fa70b5044d109ea +generated: "2023-10-10T06:58:15.158733765+02:00" diff --git a/command-service/helm-charts/spark-connector-cron/Chart.yaml b/command-service/helm-charts/spark-connector-cron/Chart.yaml new file mode 100644 index 00000000..a7a290f9 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +appVersion: 1.0.0 +dependencies: +- name: common + repository: https://nimbushubin.github.io/helmcharts + version: 0.1.0 +description: A production-ready Helm chart base template +maintainers: +- name: NimbusHub.in +name: spark-submit-cron +version: 0.1.0 diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/.helmignore b/command-service/helm-charts/spark-connector-cron/charts/common/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/Chart.yaml b/command-service/helm-charts/spark-connector-cron/charts/common/Chart.yaml new file mode 100644 index 00000000..cc1e9676 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +description: Library chart for Sunbird +keywords: +- common +- helper +- template +- function +maintainers: +- name: NimbusHub.in +name: common +type: library +version: 0.1.0 diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_affinities.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..e85b1df4 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_affinities.tpl @@ -0,0 +1,139 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a topologyKey definition +{{ include "common.affinities.topologyKey" (dict "topologyKey" "BAR") -}} +*/}} +{{- define "common.affinities.topologyKey" -}} +{{ .topologyKey | default "kubernetes.io/hostname" -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: 1 + {{- range $extraPodAffinityTerms }} + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: {{ .weight | default 1 -}} + {{- end -}} +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- range $extraPodAffinityTerms }} + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- end -}} +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_capabilities.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..b1257397 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_capabilities.tpl @@ -0,0 +1,229 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for daemonset. +*/}} +{{- define "common.capabilities.daemonset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Vertical Pod Autoscaler. +*/}} +{{- define "common.capabilities.vpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if PodSecurityPolicy is supported +*/}} +{{- define "common.capabilities.psp.supported" -}} +{{- if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if AdmissionConfiguration is supported +*/}} +{{- define "common.capabilities.admisionConfiguration.supported" -}} +{{- if semverCompare ">=1.23-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for AdmissionConfiguration. +*/}} +{{- define "common.capabilities.admisionConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiserver.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for PodSecurityConfiguration. +*/}} +{{- define "common.capabilities.podSecurityConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "pod-security.admission.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_configs.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_configs.tpl new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_errors.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..07ded6f6 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_errors.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_images.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_images.tpl new file mode 100644 index 00000000..1bcb779d --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_images.tpl @@ -0,0 +1,117 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" .Values.global ) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $separator := ":" -}} +{{- $termination := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .imageRoot.digest }} + {{- $separator = "@" -}} + {{- $termination = .imageRoot.digest | toString -}} +{{- end -}} +{{- if $registryName }} + {{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}} +{{- else -}} + {{- printf "%s%s%s" $repositoryName $separator $termination -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end }} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper image version (ingores image revision/prerelease info & fallbacks to chart appVersion) +{{ include "common.images.version" ( dict "imageRoot" .Values.path.to.the.image "chart" .Chart ) }} +*/}} +{{- define "common.images.version" -}} +{{- $imageTag := .imageRoot.tag | toString -}} +{{/* regexp from https://github.com/Masterminds/semver/blob/23f51de38a0866c5ef0bfc42b3f735c73107b700/version.go#L41-L44 */}} +{{- if regexMatch `^([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?$` $imageTag -}} + {{- $version := semver $imageTag -}} + {{- printf "%d.%d.%d" $version.Major $version.Minor $version.Patch -}} +{{- else -}} + {{- print .chart.AppVersion -}} +{{- end -}} +{{- end -}} + diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_ingress.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..efa5b85c --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_ingress.tpl @@ -0,0 +1,73 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") (hasKey .annotations "kubernetes.io/tls-acme") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_labels.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..d90a6cdc --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_labels.tpl @@ -0,0 +1,46 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Kubernetes standard labels +{{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) -}} +*/}} +{{- define "common.labels.standard" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{- $default := dict "app.kubernetes.io/name" (include "common.names.name" .context) "helm.sh/chart" (include "common.names.chart" .context) "app.kubernetes.io/instance" .context.Release.Name "app.kubernetes.io/managed-by" .context.Release.Service -}} +{{- with .context.Chart.AppVersion -}} +{{- $_ := set $default "app.kubernetes.io/version" . -}} +{{- end -}} +{{ template "common.tplvalues.merge" (dict "values" (list .customLabels $default) "context" .context) }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Labels used on immutable fields such as deploy.spec.selector.matchLabels or svc.spec.selector +{{ include "common.labels.matchLabels" (dict "customLabels" .Values.podLabels "context" $) -}} + +We don't want to loop over custom labels appending them to the selector +since it's very likely that it will break deployments, services, etc. +However, it's important to overwrite the standard labels if the user +overwrote them on metadata.labels fields. +*/}} +{{- define "common.labels.matchLabels" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{ merge (pick (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) "app.kubernetes.io/name" "app.kubernetes.io/instance") (dict "app.kubernetes.io/name" (include "common.names.name" .context) "app.kubernetes.io/instance" .context.Release.Name ) | toYaml }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_names.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_names.tpl new file mode 100644 index 00000000..a222924f --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_names.tpl @@ -0,0 +1,71 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_secrets.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a193c46b --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_secrets.tpl @@ -0,0 +1,172 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + - failOnNew - Boolean - Optional - Default to true. If set to false, skip errors adding new keys to existing secrets. +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $failOnNew := default true .failOnNew }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key | quote }} + {{- else if $failOnNew }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Reuses the value from an existing secret, otherwise sets its value to a default value. + +Usage: +{{ include "common.secrets.lookup" (dict "secret" "secret-name" "key" "keyName" "defaultValue" .Values.myValue "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - defaultValue - String - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - context - Context - Required - Parent context. + +*/}} +{{- define "common.secrets.lookup" -}} +{{- $value := "" -}} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data -}} +{{- if and $secretData (hasKey $secretData .key) -}} + {{- $value = index $secretData .key -}} +{{- else if .defaultValue -}} + {{- $value = .defaultValue | toString | b64enc -}} +{{- end -}} +{{- if $value -}} +{{- printf "%s" $value -}} +{{- end -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_storage.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..16405a0f --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_storage.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_tplvalues.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..a8ed7637 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,38 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template perhaps with scope if the scope is present. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }} +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }} +*/}} +{{- define "common.tplvalues.render" -}} +{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} +{{- if contains "{{" (toJson .value) }} + {{- if .scope }} + {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }} + {{- else }} + {{- tpl $value .context }} + {{- end }} +{{- else }} + {{- $value }} +{{- end }} +{{- end -}} + +{{/* +Merge a list of values that contains template after rendering them. +Merge precedence is consistent with http://masterminds.github.io/sprig/dicts.html#merge-mustmerge +Usage: +{{ include "common.tplvalues.merge" ( dict "values" (list .Values.path.to.the.Value1 .Values.path.to.the.Value2) "context" $ ) }} +*/}} +{{- define "common.tplvalues.merge" -}} +{{- $dst := dict -}} +{{- range .values -}} +{{- $dst = include "common.tplvalues.render" (dict "value" . "context" $.context "scope" $.scope) | fromYaml | merge $dst -}} +{{- end -}} +{{ $dst | toYaml }} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_utils.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..bfbddf05 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_utils.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ include "common.names.namespace" .context | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Checksum a template at "path" containing a *single* resource (ConfigMap,Secret) for use in pod annotations, excluding the metadata (see #18376). +Usage: +{{ include "common.utils.checksumTemplate" (dict "path" "/configmap.yaml" "context" $) }} +*/}} +{{- define "common.utils.checksumTemplate" -}} +{{- $obj := include (print .context.Template.BasePath .path) .context | fromYaml -}} +{{ omit $obj "apiVersion" "kind" "metadata" | toYaml | sha256sum }} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_variables.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_variables.tpl new file mode 100644 index 00000000..e5572991 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_variables.tpl @@ -0,0 +1,41 @@ +{{/* Define findkey function */}} +{{- define "common.variables.findkey" -}} +{{- $value := .value -}} +{{- $keys := .keys -}} +{{- $found := true -}} + +{{- range $key := $keys -}} + {{- if kindIs "map" $value -}} + {{- if hasKey $value $key -}} + {{- $value = index $value $key -}} + {{- else -}} + {{- $found = false -}} + {{- break -}} + {{- end -}} + {{- else -}} + {{- $found = false -}} + {{- break -}} + {{- end -}} +{{- end -}} + +{{- if $found -}} + {{- $value -}} +{{- else -}} + {{- "" -}} +{{- end -}} +{{- end -}} + +{{/* Define getmethekey function with precedence and input as a dot-delimited string using findkey */}} +{{- define "common.variables.variableGlobal" -}} +{{- $root := first . -}} +{{- $keyString := index (rest .) 0 -}} +{{- $keys := splitList "." $keyString -}} + +{{- $value := include "common.variables.findkey" (dict "value" $root.Values "keys" $keys) -}} + +{{- if (not $value) -}} + {{- $value = include "common.variables.findkey" (dict "value" $root.Values.global "keys" $keys) -}} +{{- end -}} + +{{- $value -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/_warnings.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..66dffc1f --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/_warnings.tpl @@ -0,0 +1,19 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_cassandra.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..eda9aada --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mariadb.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..17d83a2f --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mongodb.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..bbb445b8 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,113 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mysql.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..ca3953f8 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_postgresql.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..8c9aa570 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,134 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_redis.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..fc0d208d --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,81 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_validations.tpl b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..31ceda87 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,51 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/spark-connector-cron/charts/common/values.yaml b/command-service/helm-charts/spark-connector-cron/charts/common/values.yaml new file mode 100644 index 00000000..c83b33f2 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/charts/common/values.yaml @@ -0,0 +1,82 @@ +# Default values for common. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/command-service/helm-charts/spark-connector-cron/templates/NOTES.txt b/command-service/helm-charts/spark-connector-cron/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/spark-connector-cron/templates/_base_serviceAccount.tpl b/command-service/helm-charts/spark-connector-cron/templates/_base_serviceAccount.tpl new file mode 100644 index 00000000..14cd7f2d --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/_base_serviceAccount.tpl @@ -0,0 +1,4 @@ +{{- define "base.serviceaccountname" -}} + {{- $name := printf "%s-%s" .Chart.Name "sa" }} + {{- default $name .Values.serviceAccount.name }} +{{- end }} \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl b/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl new file mode 100644 index 00000000..f4b82f3c --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/_cron_release_name.tpl @@ -0,0 +1,4 @@ +{{- define "base.cronReleaseName" -}} + {{- $name := printf "%s--%s" .Chart.Name .Values.instance_id }} + {{- default .Values.instance_id $name }} +{{- end }} \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/templates/_image.tpl b/command-service/helm-charts/spark-connector-cron/templates/_image.tpl new file mode 100644 index 00000000..0601533c --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/_image.tpl @@ -0,0 +1,10 @@ +{{- define "base.image" }} +{{- $registry := default .Values.global.image.registry .Values.registry }} +{{- $image := printf "%s/%s" $registry .Values.repository}} +{{- if .Values.digest }} +{{- printf "%s@%s" $image .Values.digest }} +{{- else }} +{{- $tag := default "latest" .Values.tag }} +{{- printf "%s:%s" $image $tag }} +{{- end }} +{{- end }} diff --git a/command-service/helm-charts/spark-connector-cron/templates/_namespace.tpl b/command-service/helm-charts/spark-connector-cron/templates/_namespace.tpl new file mode 100644 index 00000000..4016b1e4 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/_namespace.tpl @@ -0,0 +1,14 @@ +# .Values.namespace will get overridden by .Values.global.namespace.chart-name +{{- define "base.namespace" -}} + {{- $chartName := .Chart.Name }} + {{- $namespace := default .Release.Namespace .Values.namespace }} + {{- if .Values.global }} + {{- with .Values.global.namespace }} + {{- if hasKey . $chartName }} + {{- $namespace = index . $chartName }} + {{- end }} + {{- end }} + {{- end }} + {{- $namespace | trunc 63 | trimSuffix "-" }} +{{- end }} + diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml new file mode 100644 index 00000000..f42637a4 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -0,0 +1,82 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "base.cronReleaseName" . }} + namespace: {{ include "base.namespace" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + schedule: {{ .Values.cronSchedule }} + jobTemplate: + spec: + template: + metadata: + annotations: + {{- if .Values.podAnnotations }} + {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 12 }} + {{- end }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 12 }} + spec: + serviceAccountName: {{ include "base.serviceaccountname" . }} + restartPolicy: {{ .Values.restartPolicy }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 12 }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 12 }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{- include "base.image" .}}" + imagePullPolicy: {{ .Values.imagePullPolicy }} + {{- if .Values.livenessProbe }} + livenessProbe: + {{- toYaml .Values.livenessProbe | nindent 16 }} + {{- end }} + {{- if .Values.readinessProbe }} + readinessProbe: + {{- toYaml .Values.readinessProbe | nindent 16 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 16 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 16 }} + {{- if .Values.configmap.enabled }} + envFrom: + - configMapRef: + name: {{ include "common.names.fullname" . }}-env + volumeMounts: + - name: config + mountPath: {{ .Values.configmap.mountPath }} + {{- end }} + command: + {{- if eq .Values.technology "scala" }} + - /bin/sh + - -c + - |+ + # Wait for the Spark pod to be ready + SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\*.jar --class {{ .Values.main_class }} /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-scala-config.conf -c {{ .Values.instance_id }}" + {{- else if eq .Values.technology "python" }} + - /bin/sh + - -c + - | + # Wait for the Spark pod to be ready + SPARK_POD=$(kubectl get pods -l app.kubernetes.io/name=spark,app.kubernetes.io/component=master -o jsonpath='{.items[0].metadata.name}') + kubectl exec -it $SPARK_POD -- bash -c "/opt/bitnami/spark/bin/spark-submit --master={{ .Values.spark.master.host }} --conf spark.pyspark.driver.python={{ .Values.python_path }} --conf spark.pyspark.python={{ .Values.python_path }} --jars /data/connectors/{{ .Values.connector_source }}/libs/\* /data/connectors/{{ .Values.connector_source }}/{{ .Values.main_file }} -f /data/conf/connectors-python-config.yaml -c {{ .Values.instance_id }}" + {{- end }} + {{- with .Values.sidecars }} + {{- toYaml . | nindent 12 }} + {{- end }} + {{- if .Values.configmap.enabled }} + volumes: + - name: config + configMap: + name: {{ include "common.names.fullname" . }} + {{- end }} diff --git a/command-service/helm-charts/spark-connector-cron/templates/rbac.yaml b/command-service/helm-charts/spark-connector-cron/templates/rbac.yaml new file mode 100644 index 00000000..7e291fd6 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/rbac.yaml @@ -0,0 +1,48 @@ +{{- if .Values.rbac.enabled -}} +--- +{{- if .Values.rbac.useClusterRole }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "common.names.fullname" . }} +rules: +{{- toYaml .Values.rbac.rules | nindent 2 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "common.names.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "base.serviceaccountname" . }} + namespace: {{ include "base.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "common.names.fullname" . }} +{{- else }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} +rules: +{{- toYaml .Values.rbac.rules | nindent 2 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} +subjects: +- kind: ServiceAccount + name: {{ include "base.serviceaccountname" . }} + namespace: {{ include "base.namespace" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "common.names.fullname" . }} +{{- end }} + +{{- end }} + diff --git a/command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml b/command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml new file mode 100644 index 00000000..d26e8536 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "base.serviceaccountname" . }} + namespace: {{ include "base.namespace" . }} + {{- if or .Values.serviceAccount.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceAccount.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml new file mode 100644 index 00000000..af90fa31 --- /dev/null +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -0,0 +1,166 @@ +nameOverride: "" +fullnameOverride: "spark-submit-cron" +namespace: "spark" + +replicaCount: 1 + +global: + image: + registry: "sanketikahub" + +registry: "bitnami" +repository: "kubectl" +tag: "latest" +digest: "" + +imagePullPolicy: IfNotPresent +imagePullSecrets: [] + +commonLabels: {} + +commonAnnotations: {} + +podAnnotations: {} + +podSecurityContext: {} + # runAsNonRoot: true + # runAsUser: 1001 + # fsGroup: 1001 + +securityContext: {} + # readOnlyRootFilesystem: false + # capabilities: + # drop: + # - ALL + +# This block is an interface for k8s service spec. +service: + type: ClusterIP + ports: + - name: http + port: 80 + targetPort: 80 + +ingress: + enabled: false + annotations: {} + hosts: + - paths: + - / + # host: chart-example.local + +resources: {} + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + +livenessProbe: {} + # httpGet: + # path: "/healthz" + # port: 8080 + # initialDelaySeconds: 5 + # periodSeconds: 5 + +readinessProbe: {} + # httpGet: + # path: "/ready" + # port: 8080 + # initialDelaySeconds: 5 + # periodSeconds: 5 + +nodeSelector: {} +tolerations: [] +affinity: {} + +configmap: + enabled: false + mountPath: /config + +serviceAccount: + create: true + name: spark-cron-sa + annotations: {} + +rbac: + enabled: true + useClusterRole: true + rules: + - apiGroups: [""] + resources: ["pods"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + +serviceMonitor: + enabled: false + selectorLabels: + release: monitoring + endpoints: [] + # - port: http # the name of the port in your service, assuming the primary service port is named 'http' in this example. + # path: /metrics + # interval: 30s + # scrapeTimeout: 10s + # honorLabels: true + +# Example values.yaml structure +initContainers: {} + # - name: init-myservice + # image: busybox:1.28 + # command: ['sh', '-c', "until nslookup kubernetes.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"] + +sidecars: {} + # - name: log-reader # Sidecar container + # image: busybox # Use another busybox image + # command: ["/bin/sh"] # Override the default command + # args: ["-c", "tail -f /var/log/app.log"] # Run a shell script that tails the log file + +### NOTE: This section isn't to be changed and must be in sync with the spark helm chart +######## Spark Configuration Start ######## +## Defaults taken from Bitnami Spark Helm Chart +python_path: /opt/bitnami/python/bin/python + +spark: + host: http://spark-master-svc.spark.svc.cluster.local:8998/batches/ + master: + host: spark-master-svc.spark.svc.cluster.local + port: 7077 + driver: + extraJavaOptions: -Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties + executor: + extraJavaOptions: -Dlog4j.configuration=file:/opt/bitnami/spark/conf/log4j.properties +######## Spark Configuration End ######## + + + +## JDBC Connector +technology: scala +connector_source: jdbc-connector-1.0.0 +instance_id: nyt-psql.1 +main_class: org.sunbird.obsrv.connector.JDBCConnector +main_file: jdbc-connector-1.0.0.jar + +## Object Store Connector +# technology: python +# connector-source: object_store_connector-0.1.0 +# instance-id: s3.new-york-taxi-data.1 +# main_file: object_store_connector/__main__.py + +# The schedule in cron format +cronSchedule: 0 * * * * # Every hour +restartPolicy: OnFailure \ No newline at end of file diff --git a/command-service/requirements.txt b/command-service/requirements.txt index 8781cb5f..39b67b5d 100644 --- a/command-service/requirements.txt +++ b/command-service/requirements.txt @@ -1,6 +1,20 @@ fastapi==0.103.0 uvicorn==0.20.0 dataclasses-json==0.5.7 -urllib3==2.0.7 +# pydantic==1.10.5 pyyaml==6.0.1 +urllib3==2.0.7 +# protobuf==3.20.* +# pyhelm==2.14.5 +# kubernetes==28.1.0 +# helm_operator_sdk +# psycopg2==2.9.5 +psycopg2-binary +dacite==1.8.0 backoff==2.2.1 +tenacity==8.2.2 +kafka-python-ng==2.2.2 +boto3 +prometheus-client +kubernetes +requests \ No newline at end of file diff --git a/command-service/src/command/__init__.py b/command-service/src/command/__init__.py index 56523041..7b89c048 100644 --- a/command-service/src/command/__init__.py +++ b/command-service/src/command/__init__.py @@ -1,3 +1,7 @@ +# autoflake: skip_file from .command_executor import CommandExecutor +from .connector_registry import ConnectorRegistry +from .db_command import DBCommand +from .druid_command import DruidCommand from .flink_command import FlinkCommand -from .icommand import ICommand \ No newline at end of file +from .icommand import ICommand diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py new file mode 100644 index 00000000..880fde1c --- /dev/null +++ b/command-service/src/command/alert_manager_command.py @@ -0,0 +1,213 @@ +import json + +from command.icommand import ICommand +from config import Config +from model.data_models import Action, ActionResponse, CommandPayload +from service.db_service import DatabaseService +from service.http_service import HttpService + + +class AlertManagerService(ICommand): + + def __init__( + self, config: Config, db_service: DatabaseService, http_service: HttpService + ): + self.config = config + self.db_service = db_service + self.http_service = http_service + self.metrics = self.config.find("alert_manager.metrics") + self.masterdata_metrics = self.config.find("alert_manager.masterdata_metrics") + self.object_connector_metrics = self.config.find( + "alert_manager.object_connector_metrics" + ) + self.jdbc_connector_metrics = self.config.find( + "alert_manager.jdbc_connector_metrics" + ) + self.config_service_host = self.config.find("config_service.host") + self.config_service_port = self.config.find("config_service.port") + self.base_url = ( + f"http://{self.config_service_host}:{self.config_service_port}/alerts/v1" + ) + + def execute(self, command_payload: CommandPayload, action: Action): + dataset = self.get_dataset(dataset_id=command_payload.dataset_id) + if dataset is None: + return ActionResponse( + status="ERROR", + status_code=500, + error_message=f"Dataset {command_payload.dataset_id} does not exist", + ) + + dataset_source_config = self.get_dataset_source_config( + dataset_id=command_payload.dataset_id + ) + + for metric in self.metrics: + for service, metrics in metric.items(): + for metric_info in metrics: + self.create_alert_metric( + payload=command_payload, + service=service, + metric=metric_info, + dataset_name=dataset["name"], + ) + + for config in dataset_source_config: + if config["connector_type"] == "object": + for metric in self.object_connector_metrics: + self.create_alert_metric( + payload=command_payload, + service=service, + metric=metric, + dataset_name=dataset["name"], + ) + if config["connector_type"] == "jdbc": + for metric in self.jdbc_connector_metrics: + self.create_alert_metric( + payload=command_payload, + service=service, + metric=metric, + dataset_name=dataset["name"], + ) + + if dataset["type"] == "master-dataset": + for metric_info in self.masterdata_metrics: + self.create_alert_metric( + payload=command_payload, + service=service, + metric=metric_info, + dataset_name=dataset["name"], + ) + return ActionResponse(status="OK", status_code=200) + + def get_dataset(self, dataset_id: str) -> str: + query = f"SELECT * FROM datasets WHERE dataset_id='{dataset_id}'" + result = self.db_service.execute_select_one(sql=query) + return result + + def get_dataset_source_config(self, dataset_id: str) -> str: + query = f"SELECT * FROM dataset_source_config WHERE dataset_id='{dataset_id}'" + result = self.db_service.execute_select_all(sql=query) + return result + + def get_modified_metric( + self, service: str, metric: dict, payload: CommandPayload + ) -> dict: + if service == "flink": + substring = f"{payload.dataset_id}" + modified_substring = substring.replace("-", "_") + modified_metric = metric["metric"].replace("dataset_id", modified_substring) + metric["metric"] = modified_metric + return metric + else: + metric["metric"] = metric["metric"].replace( + "dataset_id", payload.dataset_id + ) + return metric + + def create_alert_metric( + self, payload: CommandPayload, service: str, metric: dict, dataset_name: str + ) -> ActionResponse: + metric_url = f"{self.base_url}/metric/alias/create" + + metric_data = self.get_modified_metric( + service=service, metric=metric, payload=payload + ) + + prom_metric = metric_data["metric"] + metric_alias = f"{metric_data['alias']} ({payload.dataset_id})" + # Metric api payload + metric_body = json.dumps( + { + "alias": metric_alias, + "component": "datasets", + "subComponent": dataset_name, + "metric": prom_metric, + "context": { + "datasetId": payload.dataset_id, + }, + } + ) + + response = self.http_service.post( + url=metric_url, + body=metric_body, + headers={"Content-Type": "application/json"}, + ) + if response.status == 200: + self.create_alert_rule( + payload={"dataset_name": dataset_name, "metric_data": metric_data} + ) + else: + error_data = json.loads(response.body) + error_message = error_data["params"]["errmsg"] + return ActionResponse( + status="ERROR", + status_code=500, + error_message=f"Error creating alert metric {metric_alias}: {error_message}", + ) + + def create_alert_rule(self, payload: dict) -> ActionResponse: + dataset_name = payload["dataset_name"] + prom_metric = payload["metric_data"]["metric"] + description = payload["metric_data"]["description"] + metric_alias = payload["metric_data"]["alias"] + frequency = payload["metric_data"]["frequency"] + interval = payload["metric_data"]["interval"] + operator = payload["metric_data"]["operator"] + threshold = payload["metric_data"]["threshold"] + + alert_body = json.dumps( + { + "name": f"{dataset_name}_{metric_alias}", + "manager": "grafana", + "description": description + or f"Automated alert set up for dataset {dataset_name}", + "category": "datasets", + "frequency": frequency, + "interval": interval, + "context": {"alertType": "SYSTEM"}, + "labels": {"component": "obsrv"}, + "metadata": { + "queryBuilderContext": { + "category": "datasets", + "subComponent": dataset_name, + "metric": prom_metric, + "operator": operator, + "threshold": threshold, + "metricAlias": metric_alias, + } + }, + "notification": {"channels": []}, + } + ) + + response = self.http_service.post( + url=f"{self.base_url}/create", + body=alert_body, + headers={"Content-Type": "application/json"}, + ) + result = json.loads(response.body) + alert_id = result["result"]["id"] + if response.status == 200: + self.publish_alert_rule(alert_id=alert_id) + else: + error_data = json.loads(response.body) + error_message = error_data["params"]["errmsg"] + return ActionResponse( + status="ERROR", + status_code=500, + error_message=f"Error creating alert rule for {dataset_name}: {error_message}", + ) + + def publish_alert_rule(self, alert_id: str) -> ActionResponse: + endpoint = f"/publish/{alert_id}" + url = self.base_url + endpoint + try: + response = self.http_service.get(url=url) + except Exception as e: + return ActionResponse( + status="ERROR", + status_code=500, + message=f"Error publishing alert rule for {alert_id}: {e}", + ) diff --git a/command-service/src/command/command_executor.py b/command-service/src/command/command_executor.py index ea0188d6..665e4636 100644 --- a/command-service/src/command/command_executor.py +++ b/command-service/src/command/command_executor.py @@ -1,34 +1,108 @@ +import logging + +import psycopg2 +from urllib3.exceptions import MaxRetryError, NewConnectionError + +from command.alert_manager_command import AlertManagerService +from command.connector_command import ConnectorCommand +from command.dataset_command import DatasetCommand +from command.db_command import DBCommand +from command.druid_command import DruidCommand from command.flink_command import FlinkCommand -from exception.exception import FlinkHelmInstallException, FlinkConnectionException, HttpConnectionException -from service.http_service import HttpService +from command.telemetry_command import TelemetryCommand from config import Config +from model.data_models import Action, ActionResponse, CommandPayload +from service.db_service import DatabaseService +from service.http_service import HttpService +from service.telemetry_service import TelemetryService -from model.data_models import Action, CommandPayload, ActionResponse -import logging -class CommandExecutor(): +class CommandExecutor: def __init__(self): self.config_obj = Config() + self.db_service = DatabaseService() self.http_service = HttpService() - self.flink_command = FlinkCommand(config=self.config_obj, http_service=self.http_service) + self.telemetry_service = TelemetryService() + self.flink_command = FlinkCommand( + config=self.config_obj, http_service=self.http_service + ) + self.connector_command = ConnectorCommand( + config=self.config_obj, db_service=self.db_service + ) + self.druid_command = DruidCommand( + config=self.config_obj, + db_service=self.db_service, + http_service=self.http_service, + ) + self.alert_manager_command = AlertManagerService( + config=self.config_obj, + db_service=self.db_service, + http_service=self.http_service, + ) + self.dataset_command = DatasetCommand( + db_service=self.db_service, + telemetry_service=self.telemetry_service, + http_service=self.http_service, + config=self.config_obj, + ) + self.db_command = DBCommand( + db_service=self.db_service, dataset_command=self.dataset_command + ) + self.audit_event_command = TelemetryCommand( + telemetry_service=self.telemetry_service, + dataset_command=self.dataset_command, + ) self.action_commands = {} - self.action_commands[Action.RESTART_PIPELINE_JOBS.name] = self.flink_command + self.action_commands[Action.START_PIPELINE_JOBS.name] = self.flink_command + self.action_commands[Action.MAKE_DATASET_LIVE.name] = self.db_command + self.action_commands[Action.SUBMIT_INGESTION_TASKS.name] = self.druid_command + self.action_commands[Action.DEPLOY_CONNECTORS.name] = self.connector_command + self.action_commands[Action.CREATE_ALERT_METRIC.name] = ( + self.alert_manager_command + ) + self.action_commands[Action.CREATE_AUDIT_EVENT.name] = self.audit_event_command self.logger = logging.getLogger() - - def execute_command(self, payload: CommandPayload): + + def execute_command(self, payload: CommandPayload, ts: int): command = payload.command.name workflow_commands = self.get_command_workflow(command) + print(workflow_commands) for sub_command in workflow_commands: command = self.action_commands[sub_command] print(f"Executing command {sub_command}") try: - result = command.execute(command_payload=payload, action=sub_command) - except (FlinkHelmInstallException, FlinkConnectionException, HttpConnectionException) as conn_error: - self.logger.exception("Error when trying to connect to flink service...", conn_error) - result = ActionResponse(status="ERROR", status_code=500, error_message="HTTP_CONNECTION_ERROR") + if sub_command == Action.CREATE_AUDIT_EVENT.name: + command.execute(command_payload=payload, action=sub_command, ts=ts) + else: + result = command.execute( + command_payload=payload, action=sub_command + ) + except ( + ConnectionRefusedError, + MaxRetryError, + NewConnectionError, + ) as conn_error: + self.logger.exception( + "Error when trying to connect to http endpoint...", conn_error + ) + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="HTTP_CONNECTION_ERROR", + ) + return result + except psycopg2.OperationalError as db_conn_error: + self.logger.exception( + "Error when trying to connect to database...", db_conn_error + ) + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="DATABASE_CONNECTION_ERROR", + ) return result - return result + return result def get_command_workflow(self, action: Action): - return self.config_obj.find("commands.{0}.workflow".format(action)) \ No newline at end of file + return self.config_obj.find("commands.{0}.workflow".format(action)) diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py new file mode 100644 index 00000000..17d2d603 --- /dev/null +++ b/command-service/src/command/connector_command.py @@ -0,0 +1,168 @@ +import logging +import subprocess + +from command.icommand import ICommand +from config import Config +from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType +from service.db_service import DatabaseService + + +class ConnectorCommand(ICommand): + def __init__(self, config: Config, db_service: DatabaseService): + self.config = config + self.db_service = db_service + self.logger = logging.getLogger() + self.connector_job_ns = "connector-jobs" + self.connector_job_chart_dir = "{0}/connector-cron-jobs".format( + self.config.find("helm_charts_base_dir") + ) + + def execute(self, command_payload: CommandPayload, action: Action): + result = None + active_connectors = self._get_connector_details( + dataset_id=command_payload.dataset_id + ) + is_masterdata = self._get_masterdata_details( + dataset_id=command_payload.dataset_id + ) + if action == Action.DEPLOY_CONNECTORS.name: + result = self._deploy_connectors( + command_payload.dataset_id, active_connectors, is_masterdata + ) + + return result + + def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): + result = None + self._stop_connector_jobs(dataset_id, active_connectors, is_masterdata) + result = self._install_jobs(dataset_id, active_connectors, is_masterdata) + + return result + + def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata): + managed_releases = [] + connector_jar_config = self.config.find("connector_job") + masterdata_jar_config = self.config.find("masterdata_job") + for connector_type in connector_jar_config: + for release in connector_jar_config[connector_type]: + managed_releases.append(release["release_name"]) + if is_masterdata: + for release in masterdata_jar_config: + managed_releases.append(release["release_name"]) + + helm_ls_cmd = ["helm", "ls", "--namespace", self.connector_job_ns] + helm_ls_result = subprocess.run( + helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + if helm_ls_result.returncode == 0: + jobs = helm_ls_result.stdout.decode() + for job in jobs.splitlines()[1:]: + release_name = job.split()[0] + if release_name in managed_releases: + print("Uninstalling job {0}".format(release_name)) + helm_uninstall_cmd = [ + "helm", + "uninstall", + release_name, + "--namespace", + self.connector_job_ns, + ] + helm_uninstall_result = subprocess.run( + helm_uninstall_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if helm_uninstall_result.returncode == 0: + print(f"Successfully uninstalled job {release_name}...") + else: + print( + f"Error uninstalling job {release_name}: {helm_uninstall_result.stderr.decode()}" + ) + + def _install_jobs(self, dataset_id, active_connectors, is_masterdata): + result = None + for connector in active_connectors: + print("Installing connector {0}".format(connector)) + connector_jar_config = self.config.find("connector_job")[connector] + for release in connector_jar_config: + result = self._perform_install(release) + if is_masterdata: + print("Installing masterdata job") + masterdata_jar_config = self.config.find("masterdata_job") + for release in masterdata_jar_config: + result = self._perform_install(release) + return result + + def _perform_install(self, release): + err = None + result = None + release_name = release["release_name"] + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + self.connector_job_chart_dir, + "--namespace", + self.connector_job_ns, + "--create-namespace", + "--set", + "file.path={}".format(release["jar"]), + "--set", + "class.name={}".format(release["class"]), + "--set", + "job.name={}".format(release_name), + "--set", + "args={}".format(",".join(release["args"])), + "--set", + "schedule={}".format(release["schedule"]), + ] + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} deployment succeeded...") + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_HELM_INSTALLATION_EXCEPTION", + ) + print( + f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + ) + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result + + def _get_connector_details(self, dataset_id): + active_connectors = [] + rows = self.db_service.execute_select_all( + f""" + SELECT DISTINCT(connector_type) + FROM dataset_source_config + WHERE status='{DatasetStatusType.Live.name}' and connector_type IN ('object', 'jdbc')""" + ) + + for row in rows: + active_connectors.append(row[0]) + + return active_connectors + + def _get_masterdata_details(self, dataset_id): + is_masterdata = False + rows = self.db_service.execute_select_all( + f""" + SELECT * + FROM datasets + WHERE status='{DatasetStatusType.Live.name}' AND dataset_id = '{dataset_id}' AND type = 'master-dataset'""" + ) + + if len(rows) > 0: + is_masterdata = True + + return is_masterdata diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py new file mode 100644 index 00000000..cef727c6 --- /dev/null +++ b/command-service/src/command/connector_registry.py @@ -0,0 +1,367 @@ +import json +import os +import tarfile +import uuid +import zipfile +from datetime import datetime + +import requests +from fastapi import status + +from config import Config +from service.http_service import HttpService +from service.db_service import DatabaseService +from model.db_models import ConnectorRegsitryv2 + + +class RegistryResponse: + def __init__(self, status: str, message: str, statusCode: status, connector_info = None): + self.status = status + self.connector_info = connector_info + self.message = message # Ensure message is directly a string + self.statusCode = statusCode + + +class ConnectorRegistry: + def __init__(self): + self.metadata = dict() + self.db_service = DatabaseService() + self.config = Config() + self.download_path = self.config.find("connector_registry.download_path") + self.uuid = str(uuid.uuid4()) + self.extraction_path = os.path.join(self.download_path, self.uuid) + self.metadata = None + self.ui_spec = None + self.metadata_file_name = self.config.find( + "connector_registry.metadata_file_name" + ) + self.ui_spec_file_name = self.config.find("connector_registry.ui_spec_file") + self.dataset_api_url = self.config.find("dataset_api.host").strip("/") + self.pre_signed_url = self.config.find("dataset_api.pre_signed_url").strip("/") + + def register(self, rel_path: str) -> RegistryResponse: + try: + download_file_path = os.path.join(self.download_path, rel_path) + file_extension = rel_path.split(".")[-1] + + # if not os.path.exists(download_file_path): + http_service = HttpService() + print( + f"Connector Registry | Received request to register connector: {rel_path} | {self.dataset_api_url}/{self.pre_signed_url}" + ) + dataset_api_request = {"request": {"files": [rel_path], "access": "read", "type": "connector"}} + dataset_api_response = http_service.post( + url=f"{self.dataset_api_url}/{self.pre_signed_url}", + body=json.dumps(dataset_api_request), + headers={"Content-Type": "application/json"} + ) + print( + f"Connector Registry | Dataset API Response: {dataset_api_response.body}" + ) + + if dataset_api_response.status != 200: + return RegistryResponse( + status="failure", + message="dataset api failed to generate read url.", + statusCode=status.HTTP_400_BAD_REQUEST, + ) + + dataset_api_response_data = json.loads(dataset_api_response.body) + url = dataset_api_response_data.get("result", [{}])[0].get( + "preSignedUrl", None + ) + if not url: + return RegistryResponse( + status="failure", + message="dataset api failed to generate read url.", + statusCode=status.HTTP_400_BAD_REQUEST, + ) + + self.cleanup_download_path() + os.makedirs(self.download_path, exist_ok=True) + + # download the file + download_status = self.download_file(url, download_file_path) + print(f"Connector Registry | Download status: {download_status}") + if not download_status: + return RegistryResponse( + status="failure", + message="failed to download the file", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + # extract the file + os.makedirs(self.extraction_path, exist_ok=True) + extraction_status = ExtractionUtil.extract( + file=download_file_path, + extract_out_path=self.extraction_path, + ext=file_extension, + ) + if extraction_status.status.lower() == "failure": + return extraction_status + + load_metadata_status = self.load_metadata(self.extraction_path) + + # Loading of metadata + if not load_metadata_status: + return RegistryResponse( + status="failure", + message="unable to locate the metadata file", + statusCode=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) + + # check if folder exists + connector_source = f"{self.metadata['metadata']['id']}-{self.metadata['metadata']['version']}" + if not os.path.exists(os.path.join(self.extraction_path, connector_source)): + return RegistryResponse( + status="failure", + message=f"connector source folder not found; expecting {connector_source} folder inside the archive", + statusCode=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) + + load_ui_config_status = self.load_ui_metadata(self.extraction_path) + + # Loading of ui configurations + if not load_ui_config_status: + return RegistryResponse( + status="failure", + message="Unable to locate the ui configuration File", + statusCode=status.HTTP_422_UNPROCESSABLE_ENTITY, + ) + + # Process of the metadata + return self.process_metadata(rel_path, connector_source) + + except Exception as e: + print(f"Connector Registry | An error occurred: {e}") + return RegistryResponse( + status="failure", + message="Failed to register connector", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + # Method to clean up the local directory + def cleanup_download_path(self): + if os.path.exists(self.download_path): + os.system(f"rm -rf {self.download_path}/*") + + # Method to load the file data into object (ex: ui config file and metadata file) + def load_json_file(self, extract_out_path, file_name, attribute_name): + for root, dirs, files in os.walk(extract_out_path): + for name in files: + if name == file_name: + print(f"Connector Registry | {file_name} found") + with open(os.path.join(root, name)) as f: + data = json.load(f) + if not data: + print(f"Connector Registry | Empty {file_name} file") + return False + setattr(self, attribute_name, data) + return True + print(f"Connector Registry | {file_name} not found") + return False + + def load_metadata(self, extract_out_path): + return self.load_json_file( + extract_out_path, self.metadata_file_name, "metadata" + ) + + def load_ui_metadata(self, extract_out_path): + return self.load_json_file(extract_out_path, self.ui_spec_file_name, "ui_spec") + + # Method to save the metadata and ui_spec configuration into db + def process_metadata(self, rel_path, connector_source) -> RegistryResponse: + result = [] + tenant = self.metadata.get("metadata", {}).get("tenant", "") + + if tenant == "multiple": + connector_objects = self.metadata["connectors"] + for obj in connector_objects: + connector_id = obj["id"].replace(" ", "-") + registry_meta = ConnectorRegsitryv2(connector_id, + obj['name'], + 'source', + self.metadata['metadata']['category'], + self.metadata['metadata']['version'], + obj['description'], + self.metadata['metadata']['technology'], + self.metadata['metadata']['runtime'], + self.metadata['metadata']['licence'], + self.metadata['metadata']['owner'], + obj['icon'], + 'Live', + rel_path, + connector_source, + 'SYSTEM', + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + self.ui_spec[obj["id"]] if obj["id"] in self.ui_spec else {}, + 'SYSTEM', + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ) + query = self.build_insert_query(registry_meta) + success = self.execute_query(query) + if not success: + return RegistryResponse( + status="failure", + message=f"Failed to register connector {connector_id}", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + result.append(registry_meta.to_dict()) + return RegistryResponse( + status="success", + connector_info=result, + message="Connectors registered successfully", + statusCode=status.HTTP_200_OK + ) + + else: + connector_id = ( + self.metadata.get("metadata", {}).get("id", "").replace(" ", "-") + ) + registry_meta = ConnectorRegsitryv2( + connector_id, + self.metadata['metadata']['name'], + 'source', + self.metadata['metadata']['category'], + self.metadata['metadata']['version'], + self.metadata['metadata']['description'], + self.metadata['metadata']['technology'], + self.metadata['metadata']['runtime'], + self.metadata['metadata']['licence'], + self.metadata['metadata']['owner'], + self.metadata['metadata']['icon'], + 'Live', + rel_path, + connector_source, + 'SYSTEM', + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + self.ui_spec[obj["id"]] if obj["id"] in self.ui_spec else {}, + 'SYSTEM', + datetime.now().strftime("%Y-%m-%d %H:%M:%S") + ) + query = self.build_insert_query(registry_meta) + success = self.execute_query(query) + if not success: + return RegistryResponse( + status="failure", + message=f"Failed to register connector {connector_id}", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + return RegistryResponse( + status="success", + message="Connectors registered successfully", + connector_info=registry_meta, + statusCode=status.HTTP_200_OK, + ) + + def execute_query(self, query) -> bool: + try: + result = self.db_service.execute_upsert(query) + return result > 0 # Assuming the result is the number of affected rows + except Exception as e: + print( + f"Connector Registry | An error occurred during the execution of Query: {e}" + ) + return False + + # Method to download the file from blob store + def download_file(self, url, destination) -> bool: + try: + print(f"destination {destination}") + response = requests.get(url, stream=True) + response.raise_for_status() + + with open(destination, "wb") as file: + for chunk in response.iter_content(chunk_size=8192): + file.write(chunk) + + print( + f"Connector Registry | Download completed successfully. URL:{url} Destination: {destination}" + ) + return True + except requests.exceptions.HTTPError as http_err: + print( + f"Connector Registry | HTTP error occurred during the file download: {http_err}" + ) + return False + except Exception as e: + print( + f"Connector Registry | An unexpected error occurred during the file download: {e}" + ) + return False + + def build_insert_query(self, registry_meta: ConnectorRegsitryv2): + ui_spec_json = json.dumps(registry_meta.ui_spec) + return f""" INSERT INTO connector_registry (id, connector_id, name, type, category, version, description, technology, runtime, licence, owner, iconurl, status, source_url, source, ui_spec, created_by, updated_by, created_date, updated_date, live_date) VALUES + ( '{registry_meta.id}-{registry_meta.version}', '{registry_meta.id}', '{registry_meta.name}', '{registry_meta.type}', '{registry_meta.category}', '{registry_meta.version}', '{registry_meta.description}', '{registry_meta.technology}', '{registry_meta.runtime}', '{registry_meta.licence}', '{registry_meta.owner}', '{registry_meta.iconurl}', '{registry_meta.status}', '{registry_meta.source_url}', '{registry_meta.source}', '{ui_spec_json}', 'SYSTEM', 'SYSTEM', '{datetime.now()}', '{datetime.now()}', '{datetime.now()}' ) + ON CONFLICT (connector_id, version) DO UPDATE + SET id = '{registry_meta.id}-{registry_meta.version}', + name = '{registry_meta.name}', + type = '{registry_meta.type}', + category = '{registry_meta.category}', + version = '{registry_meta.version}', + description = '{registry_meta.description}', + technology = '{registry_meta.technology}', + runtime = '{registry_meta.runtime}', + licence = '{registry_meta.licence}', + owner = '{registry_meta.owner}', + iconurl = '{registry_meta.iconurl}', + status = '{registry_meta.status}', + source_url = '{registry_meta.source_url}', + source = '{registry_meta.source}', + ui_spec = '{ui_spec_json}'::jsonb, + updated_date = '{datetime.now()}' + ;; + """ + + +class ExtractionUtil: + def extract_gz(tar_path, extract_path): + with tarfile.open(tar_path, "r:*") as tar: + tar.extractall(path=extract_path) + + def extract_zip(tar_path, extract_path): + with zipfile.ZipFile(tar_path, "r") as zip_ref: + zip_ref.extractall(path=extract_path) + + # Method to extract the compressed files + def extract(file, extract_out_path, ext) -> RegistryResponse: + extraction_function = ExtractionUtil.extract_gz + + compression_types = { + "zip": ExtractionUtil.extract_zip, + } + + try: + print( + f"Connector Registry | Extracting {file} to {extract_out_path} of {ext} file type" + ) + + if ext in compression_types: + extraction_function = compression_types.get(ext) + + extraction_function(file, extract_out_path) + print(f"Connector Registry | Extraction complete for {file}") + return RegistryResponse( + status="success", + message="Extraction Successfully Completed", + statusCode=status.HTTP_200_OK, + ) + except (tarfile.TarError, zipfile.BadZipFile, OSError) as e: + print( + f"Connector Registry | An error occurred while extracting the file: {e}" + ) + return RegistryResponse( + status="failure", + message="Failed to Extract the File", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + except Exception as e: + print(f"Connector Registry | An unexpected error occurred: {e}") + return RegistryResponse( + status="failure", + message="Failed to Extract the File", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py new file mode 100644 index 00000000..edd147e8 --- /dev/null +++ b/command-service/src/command/dataset_command.py @@ -0,0 +1,95 @@ +import json +import time + +from dacite import from_dict + +from command.icommand import ICommand +from config import Config +from model.data_models import CommandPayload, DatasetStatusType +from model.db_models import DatasetsLive +from model.telemetry_models import Audit, Object, Property, Transition +from service.db_service import DatabaseService +from service.http_service import HttpService +from service.telemetry_service import TelemetryService + + +class DatasetCommand(ICommand): + def __init__( + self, + db_service: DatabaseService, + telemetry_service: TelemetryService, + http_service: HttpService, + config: Config, + ): + self.db_service = db_service + self.telemetry_service = telemetry_service + self.http_service = http_service + self.config = config + self.http_service = http_service + self.config_service_host = self.config.find("config_service.host") + self.config_service_port = self.config.find("config_service.port") + self.base_url = f"http://{self.config_service_host}:{self.config_service_port}/datasets/v1/export" + + def _get_draft_dataset_record(self, dataset_id): + query = f""" + SELECT "type", MAX(version) AS max_version FROM datasets_draft WHERE dataset_id = '{dataset_id}' GROUP BY 1 + """ + dataset_record = self.db_service.execute_select_one(query) + if dataset_record is not None: + return dataset_record + return None + + def _check_for_live_record(self, dataset_id): + query = f""" + SELECT * FROM datasets WHERE dataset_id = '{dataset_id}' AND status = '{DatasetStatusType.Live.name}' + """ + result = self.db_service.execute_select_one(query) + live_dataset = dict() + if result is not None: + live_dataset = from_dict(data_class=DatasetsLive, data=result) + data_version = live_dataset.data_version + 1 + return live_dataset, data_version + return None, None + + def audit_live_dataset(self, command_payload: CommandPayload, ts: int): + dataset_id = command_payload.dataset_id + dataset_record, data_version = self._check_for_live_record(dataset_id) + export_dataset = self.http_service.post( + url=self.base_url, + body=json.dumps({"dataset_id": dataset_id}), + headers={"Content-Type": "application/json"}, + ) + if export_dataset.status == 200: + result = json.loads(export_dataset.body) + object_ = Object( + dataset_id, dataset_record.type, dataset_record.data_version + ) + live_dataset_property = Property("dataset:export", result["result"], "") + draft_property = Property( + "draft-dataset:status", + DatasetStatusType.ReadyToPublish.name, + DatasetStatusType.Live.name, + ) + dataset_property = Property( + "dataset:status", + DatasetStatusType.Live.name, + DatasetStatusType.Live.name, + ) + transition = Transition(duration=int(time.time() - ts)) + edata = Audit( + props=[ + draft_property, + dataset_property, + live_dataset_property, + ], + transition=transition, + ) + self.telemetry_service.audit(object_=object_, edata=edata) + return True + else: + print( + "Failed to get dataset configurations from export API, dataset_id: ", + dataset_id, + ) + print("Dataset record details: ", json.dumps(dataset_record)) + return False diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py new file mode 100644 index 00000000..b428c3e5 --- /dev/null +++ b/command-service/src/command/db_command.py @@ -0,0 +1,263 @@ +import json +import time +from datetime import datetime as dt + +from dacite import from_dict + +from command.dataset_command import DatasetCommand +from command.icommand import ICommand +from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType +from model.db_models import ( + DatasetsDraft, + DatasetSourceConfigDraft, + DatasetTransformationsDraft, + DatasourcesDraft, +) +from service.db_service import DatabaseService + + +class DBCommand(ICommand): + + def __init__(self, db_service: DatabaseService, dataset_command: DatasetCommand): + self.db_service = db_service + self.dataset_command = dataset_command + + def execute(self, command_payload: CommandPayload, action: Action): + if action == Action.MAKE_DATASET_LIVE.name: + print( + f"Invoking MAKE_DATASET_LIVE command for dataset_id {command_payload.dataset_id}..." + ) + result = self._change_dataset_to_active(command_payload) + print(f"Result from MAKE_DATASET_ACTIVE {result}...") + return result + + def _change_dataset_to_active(self, command_payload: CommandPayload): + dataset_id = command_payload.dataset_id + live_dataset, data_version = self.dataset_command._check_for_live_record( + dataset_id + ) + if live_dataset is not None: + self.dataset_command.audit_live_dataset( + command_payload, int(time.time() * 1000) + ) + draft_dataset_id = self._insert_dataset_record( + dataset_id, data_version, live_dataset + ) + if draft_dataset_id: + self._insert_datasource_record(dataset_id, draft_dataset_id) + self._insert_dataset_source_config(dataset_id, draft_dataset_id) + self._insert_dataset_transformations(dataset_id, draft_dataset_id) + return ActionResponse(status="OK", status_code=200) + else: + return ActionResponse( + status="ERROR", status_code=404, error_message="DATASET_ID_NOT_FOUND" + ) + + def _insert_dataset_record(self, dataset_id, data_version, live_dataset): + query = f""" + SELECT * FROM datasets_draft + WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}') AND version = (SELECT MAX(version) + FROM datasets_draft WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}')) + """ + draft_dataset_record = self.db_service.execute_select_one(query) + draft_dataset_id = "" + if draft_dataset_record is not None: + draft_dataset = from_dict( + data_class=DatasetsDraft, data=draft_dataset_record + ) + draft_dataset_id = draft_dataset.id + current_timestamp = dt.now() + insert_query = f""" + INSERT INTO datasets(id, dataset_id, "type", name, extraction_config, validation_config, dedup_config, + denorm_config, data_schema, router_config, dataset_config, status, tags, data_version, created_by, updated_by, created_date, + updated_date, published_date) + VALUES ( + '{dataset_id}', + '{dataset_id}', + '{draft_dataset.type}', + '{draft_dataset.name}', + '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', + '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', + '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', + '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', + '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', + '{json.dumps(draft_dataset.router_config).replace("'", "''")}', + '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', + '{DatasetStatusType.Live.name}', + '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', + {draft_dataset.version}, + '{draft_dataset.created_by}', + '{draft_dataset.updated_by}', + '{current_timestamp}', + '{current_timestamp}', + '{current_timestamp}' + ) + ON CONFLICT (id) DO UPDATE + SET name = '{draft_dataset.name}', + extraction_config = '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', + validation_config = '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', + dedup_config = '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', + denorm_config = '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', + data_schema = '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', + router_config = '{json.dumps(draft_dataset.router_config).replace("'", "''")}', + dataset_config = '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', + tags = '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', + data_version = {data_version if live_dataset is not None else draft_dataset.version}, + updated_by = '{draft_dataset.updated_by}', + updated_date = '{current_timestamp}', + published_date = '{current_timestamp}', + status = '{DatasetStatusType.Live.name}'; + """ + self.db_service.execute_upsert(insert_query) + print(f"Dataset {dataset_id} record inserted successfully...") + self.db_service.execute_upsert( + f"UPDATE datasets_draft SET status = '{DatasetStatusType.Live.name}', published_date = now() WHERE id = '{draft_dataset_id}'" + ) + + return draft_dataset_id + + def _insert_datasource_record(self, dataset_id, draft_dataset_id): + result = {} + draft_datasource_record = self.db_service.execute_select_all( + f"SELECT * FROM datasources_draft WHERE dataset_id = '{draft_dataset_id}'" + ) + if draft_datasource_record is not None: + for record in draft_datasource_record: + draft_datasource = from_dict(data_class=DatasourcesDraft, data=record) + current_timestamp = dt.now() + insert_query = f""" + INSERT INTO datasources(id, datasource, dataset_id, datasource_ref, ingestion_spec, type, retention_period, + archival_policy, purge_policy, backup_config, status, created_by, updated_by, created_date, + updated_date, published_date, metadata) + VALUES ( + '{dataset_id + '_' + draft_datasource.datasource}', + '{draft_datasource.datasource}', + '{dataset_id}', + '{draft_datasource.datasource}', + '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', + '{draft_datasource.type}', + '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', + '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', + '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', + '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', + '{DatasetStatusType.Live.name}', + '{draft_datasource.created_by}', + '{draft_datasource.updated_by}', + '{current_timestamp}', + '{current_timestamp}', + '{current_timestamp}', + '{json.dumps(draft_datasource.metadata).replace("'", "''")}' + ) + ON CONFLICT (id) DO UPDATE + SET datasource_ref = '{draft_datasource.datasource}', + ingestion_spec = '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', + type = '{draft_datasource.type}', + retention_period = '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', + archival_policy = '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', + purge_policy = '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', + backup_config = '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', + updated_by = '{draft_datasource.updated_by}', + updated_date = '{current_timestamp}', + published_date = '{current_timestamp}', + metadata = '{json.dumps(draft_datasource.metadata).replace("'", "''")}', + status = '{DatasetStatusType.Live.name}'; + """ + result = self.db_service.execute_upsert(insert_query) + print( + f"Datasource {dataset_id + '_' + draft_datasource.datasource} record inserted successfully..." + ) + update_result = self.db_service.execute_upsert( + f"UPDATE datasources_draft SET status = '{DatasetStatusType.Live.name}', published_date = now() WHERE dataset_id = '{draft_dataset_id}'" + ) + return result + + def _insert_dataset_source_config(self, dataset_id, draft_dataset_id): + draft_dataset_source_config_record = self.db_service.execute_select_all( + f"SELECT * FROM dataset_source_config_draft WHERE dataset_id = '{draft_dataset_id}'" + ) + result = {} + if draft_dataset_source_config_record is not None: + for record in draft_dataset_source_config_record: + draft_dataset_source_config = from_dict( + data_class=DatasetSourceConfigDraft, data=record + ) + current_timestamp = dt.now() + insert_query = f""" + INSERT INTO dataset_source_config(id, dataset_id, connector_type, connector_config, connector_stats, + status, created_by, updated_by, created_date, updated_date, published_date) + VALUES ( + '{draft_dataset_source_config.id}', + '{dataset_id}', + '{draft_dataset_source_config.connector_type}', + '{json.dumps(draft_dataset_source_config.connector_config).replace("'", "''")}', + '{json.dumps(draft_dataset_source_config.connector_stats).replace("'", "''")}', + '{DatasetStatusType.Live.name}', + '{draft_dataset_source_config.created_by}', + '{draft_dataset_source_config.updated_by}', + '{current_timestamp}', + '{current_timestamp}', + '{current_timestamp}' + ) + ON CONFLICT (id) DO UPDATE + SET connector_type = '{draft_dataset_source_config.connector_type}', + connector_config = '{json.dumps(draft_dataset_source_config.connector_config).replace("'", "''")}', + connector_stats = '{json.dumps(draft_dataset_source_config.connector_stats).replace("'", "''")}', + updated_by = '{draft_dataset_source_config.updated_by}', + updated_date = '{current_timestamp}', + published_date = '{current_timestamp}', + status = '{DatasetStatusType.Live.name}'; + """ + result = self.db_service.execute_upsert(insert_query) + print( + f"Dataset Source Config {dataset_id + '_' + draft_dataset_source_config.connector_type} record inserted successfully..." + ) + update_result = self.db_service.execute_upsert( + f"UPDATE dataset_source_config_draft SET status = '{DatasetStatusType.Live.name}', published_date = now() WHERE dataset_id = '{draft_dataset_id}'" + ) + return result + + def _insert_dataset_transformations(self, dataset_id, draft_dataset_id): + draft_dataset_transformations_record = self.db_service.execute_select_all( + f"SELECT * FROM dataset_transformations_draft WHERE dataset_id = '{draft_dataset_id}'" + ) + result = {} + current_timestamp = dt.now() + if draft_dataset_transformations_record is not None: + for record in draft_dataset_transformations_record: + draft_dataset_transformations = from_dict( + data_class=DatasetTransformationsDraft, data=record + ) + insert_query = f""" + INSERT INTO dataset_transformations(id, dataset_id, field_key, transformation_function, + status, mode, created_by, updated_by, created_date, updated_date, published_date, metadata) + VALUES ( + '{dataset_id + '_' + draft_dataset_transformations.field_key}', + '{dataset_id}', + '{draft_dataset_transformations.field_key}', + '{json.dumps(draft_dataset_transformations.transformation_function).replace("'", "''")}', + '{DatasetStatusType.Live.name}', + '{draft_dataset_transformations.mode}', + '{draft_dataset_transformations.created_by}', + '{draft_dataset_transformations.updated_by}', + '{current_timestamp}', + '{current_timestamp}', + '{current_timestamp}', + '{json.dumps(draft_dataset_transformations.metadata).replace("'", "''")}' + ) + ON CONFLICT (id) DO UPDATE + SET transformation_function = '{json.dumps(draft_dataset_transformations.transformation_function).replace("'", "''")}', + updated_by = '{draft_dataset_transformations.updated_by}', + updated_date = '{current_timestamp}', + published_date = '{current_timestamp}', + metadata = '{json.dumps(draft_dataset_transformations.metadata).replace("'", "''")}', + mode = '{draft_dataset_transformations.mode}', + status = '{DatasetStatusType.Live.name}'; + """ + result = self.db_service.execute_upsert(insert_query) + print( + f"Dataset Transformation {dataset_id + '_' + draft_dataset_transformations.field_key} record inserted successfully..." + ) + update_result = self.db_service.execute_upsert( + f"UPDATE dataset_transformations_draft SET status = '{DatasetStatusType.Live.name}', published_date = now() WHERE dataset_id = '{draft_dataset_id}'" + ) + return result diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py new file mode 100644 index 00000000..1bd95ab2 --- /dev/null +++ b/command-service/src/command/druid_command.py @@ -0,0 +1,52 @@ +import json + +from command.icommand import ICommand +from config import Config +from model.data_models import Action, ActionResponse, CommandPayload +from service.db_service import DatabaseService +from service.http_service import HttpService + + +class DruidCommand(ICommand): + + def __init__( + self, config: Config, db_service: DatabaseService, http_service: HttpService + ): + self.config = config + self.db_service = db_service + self.http_service = http_service + router_host = self.config.find("druid.router_host") + router_post = self.config.find("druid.router_port") + self.supervisor_endpoint = self.config.find("druid.supervisor_endpoint") + self.router_url = f"{router_host}:{router_post}/druid" + + def execute(self, command_payload: CommandPayload, action: Action): + if action == Action.SUBMIT_INGESTION_TASKS.name: + response = self._submit_ingestion_task(command_payload.dataset_id) + return response + + def _submit_ingestion_task(self, dataset_id): + datasources_records = self.db_service.execute_select_all( + f"SELECT dso.*, dt.type as dataset_type FROM datasources dso, datasets dt WHERE dso.dataset_id = '{dataset_id}' AND dso.dataset_id = dt.id" + ) + if datasources_records is not None: + print( + f"Invoking SUBMIT_INGESTION_TASKS command for dataset_id {dataset_id}..." + ) + for record in datasources_records: + if record["dataset_type"] == "dataset" and record["type"] == "druid": + print(f"Submitting ingestion task for datasource ...") + ingestion_spec = json.dumps(record["ingestion_spec"]) + response = self.http_service.post( + url=f"{self.router_url}/{self.supervisor_endpoint}", + body=ingestion_spec, + headers={"Content-Type": "application/json"}, + ) + return ActionResponse(status="OK", status_code=200) + else: + print( + f"Dataset ID {dataset_id} not found for druid ingestion task submit..." + ) + return ActionResponse( + status="ERROR", status_code=404, error_message="DATASET_ID_NOT_FOUND" + ) diff --git a/command-service/src/command/flink_command.py b/command-service/src/command/flink_command.py index c3397923..293f54dc 100644 --- a/command-service/src/command/flink_command.py +++ b/command-service/src/command/flink_command.py @@ -1,9 +1,11 @@ +import logging +import subprocess + from command.icommand import ICommand -from model.data_models import CommandPayload, Action, ActionResponse from config import Config +from model.data_models import Action, ActionResponse, CommandPayload from service.http_service import HttpService -import subprocess -import logging + class FlinkCommand(ICommand): @@ -14,29 +16,51 @@ def __init__(self, config: Config, http_service: HttpService): def execute(self, command_payload: CommandPayload, action: Action): result = None - print(f"Invoking RESTART_PIPELINE_JOBS command...") - result = self._restart_jobs() + if action == Action.START_PIPELINE_JOBS.name: + print( + f"Invoking START_PIPELINE_JOBS command for dataset_id {command_payload.dataset_id}..." + ) + result = self._restart_jobs() return result def _restart_jobs(self): return self._install_flink_jobs() - + + def _restart_pods(self, release_name, namespace, job_name): + restart_cmd = f"kubectl delete pods --selector app=flink,component={release_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app=flink,component={release_name}-taskmanager --namespace {namespace}".format( + namespace=namespace, release_name=release_name + ) + # Run the helm command + helm_install_result = subprocess.run( + restart_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) + if helm_install_result.returncode == 0: + print(f"Job {job_name} re-deployment succeeded...") + return True + else: + print( + f"Error re-installing job {job_name}: {helm_install_result.stderr.decode()}" + ) + return False + def _install_flink_jobs(self): - namespace = self.config.find("flink.namespace") result = ActionResponse(status="OK", status_code=200) - flink_jobs = self.config.find("flink.jobs") + namespace = self.config.find("flink.namespace") for job in flink_jobs: release_name = job["release_name"] job_name = job["name"] - restart_cmd = f"kubectl delete pods --selector app=flink,component={release_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app=flink,component={release_name}-taskmanager --namespace {namespace}".format(namespace=namespace, release_name=release_name) - # Run the helm command - helm_install_result = subprocess.run(restart_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,) - - if helm_install_result.returncode == 0: - print(f"Job {job_name} re-deployment succeeded...") - else: - print(f"Error re-installing job {job_name}: {helm_install_result.stderr.decode()}") - result = ActionResponse(status="ERROR", status_code=500, error_message="FLINK_HELM_INSTALLATION_EXCEPTION") - + # Restart pods + status = self._restart_pods( + release_name=release_name, namespace=namespace, job_name=job_name + ) + if not status: + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_HELM_INSTALLATION_EXCEPTION", + ) return result diff --git a/command-service/src/command/icommand.py b/command-service/src/command/icommand.py index 6271b933..66938fd0 100644 --- a/command-service/src/command/icommand.py +++ b/command-service/src/command/icommand.py @@ -1,7 +1,7 @@ from model.data_models import Action, CommandPayload + class ICommand: def execute(self, command_payload: CommandPayload, action: Action): pass - diff --git a/command-service/src/command/telemetry_command.py b/command-service/src/command/telemetry_command.py new file mode 100644 index 00000000..d16d7abb --- /dev/null +++ b/command-service/src/command/telemetry_command.py @@ -0,0 +1,59 @@ +import time + +from command.dataset_command import DatasetCommand +from command.icommand import ICommand +from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType +from model.telemetry_models import Audit, Object, Property, Transition +from service.telemetry_service import TelemetryService + + +class TelemetryCommand(ICommand): + def __init__( + self, telemetry_service: TelemetryService, dataset_command: DatasetCommand + ): + self.dataset_command = dataset_command + self.telemetry_service = telemetry_service + + def execute(self, command_payload: CommandPayload, action: Action, ts: int): + if action == Action.CREATE_AUDIT_EVENT.name: + print( + f"Invoking CREATE_AUDIT_EVENT command for dataset_id {command_payload.dataset_id}..." + ) + object_, audit, error = self._create_audit_event(command_payload, ts) + if error: + return error + self.telemetry_service.audit(object_=object_, edata=audit) + return ActionResponse(status="OK", status_code=200) + + def _create_audit_event(self, command_payload: CommandPayload, ts: int): + dataset_id = command_payload.dataset_id + dataset_record = self.dataset_command._get_draft_dataset_record(dataset_id) + if dataset_record: + object_ = Object( + dataset_id, dataset_record["type"], dataset_record["max_version"] + ) + draft_property = Property( + "draft-dataset:status", + DatasetStatusType.ReadyToPublish.name, + DatasetStatusType.Live.name, + ) + dataset_property = Property( + "dataset:status", + DatasetStatusType.ReadyToPublish.name, + DatasetStatusType.Live.name, + ) + transition = Transition(duration=int(time.time() - ts)) + edata = Audit( + props=[draft_property, dataset_property], transition=transition + ) + return object_, edata, None + else: + return ( + None, + None, + ActionResponse( + status="ERROR", + status_code=404, + error_message="DATASET_ID_NOT_FOUND", + ), + ) diff --git a/command-service/src/config/__init__.py b/command-service/src/config/__init__.py index 3558f420..e0db2235 100644 --- a/command-service/src/config/__init__.py +++ b/command-service/src/config/__init__.py @@ -1 +1,2 @@ -from .config import Config \ No newline at end of file +# autoflake: skip_file +from .config import Config diff --git a/command-service/src/config/config.py b/command-service/src/config/config.py index d931dda4..e96475a2 100644 --- a/command-service/src/config/config.py +++ b/command-service/src/config/config.py @@ -1,13 +1,22 @@ -import yaml -from functools import reduce import operator +import os +from functools import reduce + +import yaml + +# import json +# from yaml.loader import SafeLoader -class Config(): +class Config: def __init__(self): - with open('config/service_config.yml') as config_file: + config_file = os.getenv("CONFIG_PATH", "config") + with open(os.path.join(config_file, "service_config.yml")) as config_file: self.config = yaml.safe_load(config_file) def find(self, path): element_value = reduce(operator.getitem, path.split("."), self.config) - return element_value \ No newline at end of file + # if (isinstance(element_value, list)): + # element_value = json.dumps(element_value) + # print("element_value = {0}".format(element_value)) + return element_value diff --git a/command-service/src/config/pii_rules.yml b/command-service/src/config/pii_rules.yml new file mode 100644 index 00000000..6947f9ad --- /dev/null +++ b/command-service/src/config/pii_rules.yml @@ -0,0 +1,118 @@ +keys: + address: + rule: '\baddress\b|\blocation\b|\bloc\b|\baddr\b|\badd\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m001 + financial: + rule: '\bcredit\b|\bdebit\b|\baccount\b|\btransaction\b|\btxn\b|\biban\b|\bswift\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m002 + id: + rule: '\bid\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m003 + internet: + rule: '\bemail\b|\bip\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m004 + phone: + rule: '\bphone\b|\bnumber\b|\bno\b|\bno\.\b|\bnum\b|\bmobile\b|\bcontact\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m005 + name: + rule: '\bname\b' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m006 +values: + address: + india: + rule: '(?:\bAN\b|\bAP\b|\bAD\b|\bAR\b|\bAS\b|\bBH\b|\bCH\b|\bCT\b|\bDN\b|\bDL\b|\bGA\b|\bGJ\b|\bHR\b|\bHP\b|\bJK\b|\bJH\b|\bKA\b|\bKL\b|\bLD\b|\bMP\b|\bMH\b|\bMN\b|\bME\b|\bMI\b|\bNL\b|\bOR\b|\bPY\b|\bPB\b|\bRJ\b|\bSK\b|\bTN\b|\bTS\b|\bTR\b|\bUP\b|\bUT\b|\bWB\b)' + locale: 'India' + code: 'en' + resourceKey: pii.descriptions.m007 + us: + rule: '\d{1,4} [\w\s]{1,20}(?:\bstreet\b|\bst\b|\bavenue\b|\bave\b|\broad\b|\brd\b|\bhighway\b|\bhwy\b|\bsquare\b|\bsq\b|\btrail\b|\btrl\b|\bdrive\b|\bdr\b|\bcourt\b|\bct\b|\bpark\b|\bparkway\b|\bpkwy\b|\bcircle\b|\bcir\b|\bboulevard\b|\bblvd\b)\W?(?=\s|$)' + locale: 'USA' + code: 'en' + resourceKey: pii.descriptions.m008 + financial: + credit_card: + rule: '((?:(?:\\d{4}[- ]?){3}\\d{4}|\\d{15,16}))(?![\\d])' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m009 + iban_number: + rule: '[A-Z]{2}\d{2}[A-Z0-9]{4}\d{7}([A-Z\d]?){0,16}' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m010 + id: + aadhaar: + rule: '\b\d{4}[ -]\d{4}[ -]\d{4}' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m011 + pan: + rule: '\b[A-Z]{4}\d{4}[A-Z]' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m012 + ssn: + rule: '(?:\d{3}-\d{2}-\d{4})' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m013 + internet: + email: + rule: '(?i)([A-Za-z0-9!#$%&*+\/=?^_{|.}~-]+@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m014 + ipv4: + rule: &IPv4 '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)' + locale: '' + code: 'en' + resourceKey: pii.descriptions.m015 + ipv6: + rule: &IPv6 '\s*(?!.*::.*::)(?:(?!:)|:(?=:))(?:[0-9a-f]{0,4}(?:(?<=::)|(? None: + self.node_query_response_time = Gauge( + name="node_query_response_time", + documentation="The average response time for database queries", + labelnames=["entity", "id", "endpoint", "datasetId", "status"], + registry=registry, + ) + self.total_api_calls = Counter( + name="node_total_api_calls", + documentation="The total number of API calls made", + labelnames=["entity", "id", "endpoint", "datasetId"], + registry=registry, + ) + self.failed_api_calls = Counter( + name="node_failed_api_calls", + documentation="The total number of failed API calls made", + labelnames=["entity", "id", "endpoint", "datasetId", "status"], + registry=registry, + ) + self.success_api_calls = Counter( + name="node_success_api_calls", + documentation="The total number of successful API calls made", + labelnames=["entity", "id", "endpoint", "datasetId", "status"], + registry=registry, + ) + + def queryResponseTimeMetric(self): + return self.node_query_response_time + + def totalApiCallsMetric(self): + return self.total_api_calls + + def failedApiCallsMetric(self): + return self.failed_api_calls + + def successApiCallsMetric(self): + return self.success_api_calls diff --git a/command-service/src/model/__init__.py b/command-service/src/model/__init__.py index d336e01c..5b591820 100644 --- a/command-service/src/model/__init__.py +++ b/command-service/src/model/__init__.py @@ -1 +1,10 @@ -from .data_models import Command, Action, CommandPayload, Request, ResponseParams, Result, HttpResponse \ No newline at end of file +# autoflake: skip_file +from .data_models import ( + Action, + Command, + CommandPayload, + HttpResponse, + Request, + ResponseParams, + Result, +) diff --git a/command-service/src/model/data_models.py b/command-service/src/model/data_models.py index 910afca1..198d649b 100644 --- a/command-service/src/model/data_models.py +++ b/command-service/src/model/data_models.py @@ -1,34 +1,63 @@ +import time from dataclasses import dataclass -from dataclasses_json import dataclass_json from enum import Enum +from typing import List + +from dataclasses_json import dataclass_json + class Command(Enum): + PUBLISH_DATASET = "PUBLISH_DATASET" RESTART_PIPELINE = "RESTART_PIPELINE" + class Action(Enum): - RESTART_PIPELINE_JOBS = "RESTART_PIPELINE_JOBS" + SUBMIT_INGESTION_TASKS = "SUBMIT_INGESTION_TASKS" + MAKE_DATASET_LIVE = "MAKE_DATASET_LIVE" + MAKE_DATASOURCE_ACTIVE = "MAKE_DATASOURCE_ACTIVE" + START_PIPELINE_JOBS = "START_PIPELINE_JOBS" + CREATE_ALERT_METRIC = "CREATE_ALERT_METRIC" + DEPLOY_CONNECTORS = "DEPLOY_CONNECTORS" + CREATE_AUDIT_EVENT = "CREATE_AUDIT_EVENT" + + +class DatasetStatusType(Enum): + Draft = "Draft" + Publish = "Publish" + ReadyToPublish = "ReadyToPublish" + Live = "Live" + Retired = "Retired" + Purged = "Purged" + @dataclass_json @dataclass -class CommandPayload(): +class CommandPayload: + dataset_id: str command: Command + @dataclass_json @dataclass class Request: data: CommandPayload id: str + @dataclass_json @dataclass class ResponseParams: status: str + resmsgid: str | None = None + @dataclass_json @dataclass class Result: + dataset_id: str message: str + @dataclass_json @dataclass class Response: @@ -39,11 +68,13 @@ class Response: ts: str | None = None params: ResponseParams | None = None + @dataclass_json @dataclass class HttpResponse: status: int - body: str + body: str + @dataclass class ActionResponse: @@ -51,3 +82,73 @@ class ActionResponse: status_code: int error_message: str = None + +@dataclass +class DatasetRequest: + id: str + dataset_id: str + data: List[dict] + + +@dataclass +class PIIReason: + code: str + resourceKey: str + region: str | None = None + score: float | None = None + + +@dataclass +class PIIError: + errorCode: int + errorMsg: str + errorTrace: str + + +@dataclass +class PIIResult: + field: str + type: str + score: float + reason: List[PIIReason] + + +@dataclass +class DatasetResponse: + id: str + response_code: str + status_code: int + result: List[PIIResult] | PIIError + ts: str | None = None + params: ResponseParams | None = None + + +class PIIModel: + def __init__(self): + pass + + def detect_pii(self, entity, field, value) -> PIIResult: + return PIIResult() + + def detect_entity(self, entity, value) -> List[PIIReason]: + return [PIIReason()] + + def detect_entity_in_fieldname(self, entity, value) -> List[PIIReason]: + return [PIIReason()] + + +class ParamsModels: + def __init__(self): + self.status = "" + self.msgid = "" + + def to_dict(self): + return {"status": self.status, "msgid": self.msgid} + + +class ConnectorResponseModel: + def __init__(self, id: str, ver: str): + self.id = id + self.ver = ver + self.ts = str(time.time() * 1000) + self.params = ParamsModels() diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py new file mode 100644 index 00000000..1999e8b5 --- /dev/null +++ b/command-service/src/model/db_models.py @@ -0,0 +1,127 @@ +from dataclasses import dataclass +from datetime import datetime + +from dataclasses_json import dataclass_json + + +@dataclass +class DatasetsLive: + id: str + dataset_id: str + data_version: int + type: str + name: str + validation_config: dict + extraction_config: dict + dedup_config: dict + data_schema: dict + router_config: dict + dataset_config: dict + status: str + created_by: str + created_date: datetime + tags: list[str] | None = None + updated_by: str | None = None + updated_date: datetime | None = None + denorm_config: dict | None = None + published_date: datetime | None = None + + +@dataclass +class DatasetsDraft: + id: str + dataset_id: str + version: int + type: str + name: str + validation_config: dict + extraction_config: dict + dedup_config: dict + data_schema: dict + router_config: dict + dataset_config: dict + status: str + created_by: str + created_date: datetime + tags: list[str] | None = None + updated_by: str | None = None + updated_date: datetime | None = None + denorm_config: dict | None = None + published_date: datetime | None = None + + +@dataclass +class DatasourcesDraft: + id: str + datasource: str + dataset_id: str + ingestion_spec: dict + type: str + datasource_ref: str + status: str + created_by: str + created_date: datetime + retention_period: dict | None = None + archival_policy: dict | None = None + purge_policy: dict | None = None + backup_config: dict | None = None + metadata: dict | None = None + updated_by: str | None = None + updated_date: datetime | None = None + published_date: datetime | None = None + + +@dataclass +class DatasetSourceConfigDraft: + id: str + dataset_id: str + connector_type: str + status: str + created_by: str + created_date: datetime + connector_config: dict | None = None + connector_stats: dict | None = None + updated_by: str | None = None + updated_date: datetime | None = None + published_date: datetime | None = None + + +@dataclass +class DatasetTransformationsDraft: + id: str + dataset_id: str + field_key: str + transformation_function: dict + status: str + mode: str + created_by: str + created_date: datetime + updated_by: str | None = None + updated_date: datetime | None = None + published_date: datetime | None = None + metadata: dict | None = None + + +@dataclass_json +@dataclass +class ConnectorRegsitryv2: + id: str + name: str + type: str + category: str + version: str + description: str + technology: str + runtime: str + licence: str + owner: str + iconurl: str + status: str + source_url: str + source: str + created_by: str + created_date: str + updated_date: str + ui_spec: dict | None = None + updated_by: str | None = None + livedate: datetime | None = None diff --git a/command-service/src/model/telemetry_models.py b/command-service/src/model/telemetry_models.py new file mode 100644 index 00000000..b2fb0d43 --- /dev/null +++ b/command-service/src/model/telemetry_models.py @@ -0,0 +1,75 @@ +import time +from dataclasses import dataclass +from uuid import uuid4 + +from dataclasses_json import dataclass_json + +from .data_models import DatasetStatusType + + +@dataclass_json +@dataclass +class Actor: + id: str = "SYSTEM" + type: str = "User" + + +@dataclass_json +@dataclass +class Pdata: + id: str + ver: str | None = "1.0.0" + + +@dataclass_json +@dataclass +class Context: + pdata: Pdata | None + env: str + sid: str = uuid4() + + +@dataclass_json +@dataclass +class Object: + id: str + type: str + ver: str | None + + +@dataclass_json +@dataclass +class Property: + property: str + ov: str + nv: str + + +@dataclass_json +@dataclass +class Transition: + duration: str + fromState: str | None = DatasetStatusType.ReadyToPublish.name + toState: str | None = DatasetStatusType.Live.name + timeunit: str = "milliseconds" + + +@dataclass_json +@dataclass +class Audit: + props: list[Property] + transition: Transition + action: str = "dataset:publish" + + +@dataclass_json +@dataclass +class Telemetry: + actor: Actor + context: Context + object: Object | None + edata: Audit | None + eid: str = "AUDIT" + ets: int = time.time() * 1000 + ver: str = "1.0.0" + mid: str = uuid4() diff --git a/command-service/src/routes.py b/command-service/src/routes.py index 6365fd04..7ee2839d 100644 --- a/command-service/src/routes.py +++ b/command-service/src/routes.py @@ -1,30 +1,259 @@ -from fastapi import FastAPI, status -from command.command_executor import CommandExecutor -from model.data_models import Request, Response, Result, ActionResponse +import time +import uuid from datetime import datetime as dt +from typing import List + +from fastapi import FastAPI +from fastapi import Request as FastAPIRequest +from fastapi import Response, status +from fastapi.responses import JSONResponse, PlainTextResponse +from prometheus_client import CollectorRegistry, generate_latest + +from command.command_executor import CommandExecutor +from command.connector_registry import ConnectorRegistry +from metrics import Helper +from model.data_models import ( + ActionResponse, + ConnectorResponseModel, + DatasetRequest, + DatasetResponse, + PIIError, + PIIResult, + Request, + Response, + Result, +) +from service.detect_pii_service import DetectPIIService +from service.prometheus_backup import PrometheusBackup app = FastAPI() command_executor = CommandExecutor() +pii_service = DetectPIIService() +registry = CollectorRegistry() +helper = Helper(registry) + +system_dataset_endpoint = "/system/v1/dataset/command" -@app.post("/system/dataset/command") -async def restart_pipeline(request: Request): + +@app.post(system_dataset_endpoint) +async def publish_dataset(request: Request): + start_time = int(time.time() * 1000) data = request.data - result: ActionResponse = command_executor.execute_command(payload=data) + helper.onRequest( + entity="dataset", + id=request.id, + endpoint=system_dataset_endpoint, + dataset_id=data.dataset_id, + ) + result: ActionResponse = command_executor.execute_command( + payload=data, ts=start_time + ) if result.status_code == status.HTTP_404_NOT_FOUND: - response = get_response_object(request_id=request.id, response_code="ERROR", - status_code=status.HTTP_404_NOT_FOUND, message=result.error_message) + helper.onFailedRequest( + entity="dataset", + id=request.id, + endpoint=system_dataset_endpoint, + dataset_id=data.dataset_id, + status=404, + ) + response = get_response_object( + dataset_id=data.dataset_id, + request_id=request.id, + response_code="ERROR", + status_code=status.HTTP_404_NOT_FOUND, + message=result.error_message, + ) elif result.status_code == status.HTTP_500_INTERNAL_SERVER_ERROR: - response = get_response_object(request_id=request.id, response_code="ERROR", - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, message=result.error_message) + helper.onFailedRequest( + entity="dataset", + id=request.id, + endpoint=system_dataset_endpoint, + dataset_id=data.dataset_id, + status=500, + ) + response = get_response_object( + dataset_id=data.dataset_id, + request_id=request.id, + response_code="ERROR", + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + message=result.error_message, + ) else: - response = get_response_object(request_id=request.id, response_code="OK", - status_code=status.HTTP_200_OK, message="RESTARTING_PIPELINE_SUCCESSFUL") + helper.onSuccessRequest( + entity="dataset", + id=request.id, + endpoint=system_dataset_endpoint, + dataset_id=data.dataset_id, + ) + response = get_response_object( + dataset_id=data.dataset_id, + request_id=request.id, + response_code="OK", + status_code=status.HTTP_200_OK, + message="PUBLISH_DATASET_SUCCESSFUL", + ) return response -def get_response_object(request_id, response_code, status_code, message=None): - response = Response(id=request_id, - response_code=response_code, - status_code=status_code, - ts=dt.now().strftime("%Y-%m-%d %H:%M:%S"), - result=Result(message=message)) + +def get_response_object( + dataset_id, request_id, response_code, status_code, message=None +): + response = Response( + id=request_id, + response_code=response_code, + status_code=status_code, + ts=dt.now().strftime("%Y-%m-%d %H:%M:%S"), + result=Result(dataset_id=dataset_id, message=message), + ) return response + + +pii_endpoint = "/system/data/v1/analyze/pii" + + +@app.post(pii_endpoint) +def analyze_pii(request: DatasetRequest) -> DatasetResponse: + helper.onRequest( + entity="dataset", id=request.id, endpoint=pii_endpoint, dataset_id=None + ) + try: + event_data = request.data[0] + except Exception as err: + result: PIIError = { + "errorCode": 500, + "errorMsg": type(err), + "errorTrace": err.args, + } + else: + result: List[PIIResult] | PIIError = pii_service.detect_pii_fields(event_data) + finally: + if type(result) == list: + helper.onSuccessRequest( + entity="dataset", id=request.id, endpoint=pii_endpoint, dataset_id=None + ) + response_code = "OK" + status_code = 200 + else: + helper.onFailedRequest( + entity="dataset", + id=request.id, + endpoint=pii_endpoint, + dataset_id=None, + status=500, + ) + response_code = "INTERNAL_SERVER_ERROR" + status_code = 500 + response: DatasetResponse = { + "id": str(uuid.uuid4()), + "response_code": response_code, + "status_code": status_code, + "result": result, + "ts": str(time.time() * 1000), + "params": {"status": "ACTIVE"}, + } + return response + + +@app.get("/metrics", response_class=PlainTextResponse) +def metrics(): + data = generate_latest(registry=registry) + return data + + +connector_registry_endpoint = "/connector/v1/register" + + +@app.post(connector_registry_endpoint) +async def register_connector(req: FastAPIRequest): + response = ConnectorResponseModel(id="api.connector.registry", ver="1.0") + try: + data = await req.json() + response.params.msgid = data.get("params", {}).get("msgid", str(uuid.uuid4())) + + # Get the connector relative path in data + rel_path: str = data.get("relative_path", None) + + print( + f"Connector Registry | Received request to register connector: {rel_path}" + ) + + if not rel_path: + response.params.status = "Failure" + return JSONResponse( + { + "id": response.id, + "ver": response.ver, + "ts": response.ts, + "params": response.params.to_dict(), + "responseCode": status.HTTP_400_BAD_REQUEST, + "error": {"message": "connector relative path is missing."}, + }, + status_code=status.HTTP_400_BAD_REQUEST, + ) + + executor = ConnectorRegistry() + result = executor.register(rel_path) + response.params.status = result.status + + content = { + "id": response.id, + "ver": response.ver, + "ts": response.ts, + "params": response.params.to_dict(), + "responseCode": result.status, + } + + print(f"Connector Registry | Connector registration status: {content}") + + if result.status == "success": + content["message"] = "connector registered successfully." + content["connector_info"] = result.connector_info + else: + content["error"] = {"message": result.message} + + return JSONResponse(content=content, status_code=result.statusCode) + + except Exception as e: + print(f"Connector Registry | An error occurred while Calling API: {e}") + return JSONResponse( + content={ + "id": response.id, + "ver": response.ver, + "ts": response.ts, + "params": response.params.to_dict(), + "responseCode": status.HTTP_500_INTERNAL_SERVER_ERROR, + "error": {"message": str(e)}, + }, + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + +prom_backup_endpoint = "/system/v1/backup/prometheus" + + +@app.get(prom_backup_endpoint) +def backup_prometheus() -> PlainTextResponse: + helper.onRequest( + entity="prometheus_backup", + id=None, + endpoint=prom_backup_endpoint, + dataset_id=None, + ) + prometheus_backup = PrometheusBackup() + prometheus_backup.backup_prometheus() + helper.onSuccessRequest( + entity="prometheus_backup", + id=None, + endpoint=prom_backup_endpoint, + dataset_id=None, + ) + return "Prometheus backup successful" + + +if __name__ == "__main__": + import uvicorn + + print("Starting server") + config = uvicorn.Config("routes:app", port=8000, log_level="info") + server = uvicorn.Server(config) + server.run() diff --git a/command-service/src/service/__init__.py b/command-service/src/service/__init__.py index 6ae50b6d..521c0d1d 100644 --- a/command-service/src/service/__init__.py +++ b/command-service/src/service/__init__.py @@ -1 +1,3 @@ -from .http_service import HttpService \ No newline at end of file +# autoflake: skip_file +from .db_service import DatabaseService +from .http_service import HttpService diff --git a/command-service/src/service/backup_service.py b/command-service/src/service/backup_service.py new file mode 100644 index 00000000..f7fb8b54 --- /dev/null +++ b/command-service/src/service/backup_service.py @@ -0,0 +1,152 @@ +import os +import subprocess +import tarfile +import time + +import boto3 + +# from google.cloud import storage as gcs_storage +# from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient + + +class BackupService: + def __init__(self, pod_name, namespace="default"): + self.pod_name = pod_name + self.namespace = namespace + + def compress_file(self, file_path): + timestamp = time.strftime("%Y%m%d%H%M%S") + tar_file_path = f"{file_path}_{timestamp}.tar.gz" + with tarfile.open(tar_file_path, "w:gz") as tar: + tar.add(file_path, arcname=os.path.basename(file_path)) + return tar_file_path + + def compress_remote_path(self, remote_path): + timestamp = time.strftime("%Y%m%d%H%M%S") + tar_file_path = f"{remote_path}_{timestamp}.tar.gz" + compress_command = f"tar -czvf {tar_file_path} {os.path.dirname(remote_path)}/{os.path.basename(remote_path)}" + cmd = [ + "kubectl", + "exec", + self.pod_name, + "-n", + self.namespace, + "--", + "sh", + "-c", + compress_command, + ] + subprocess.run(cmd, check=True) + return tar_file_path + + def copy_to_pod(self, local_file_path, remote_file_path): + cmd = [ + "kubectl", + "cp", + local_file_path, + f"{self.pod_name}:{remote_file_path}", + "-n", + self.namespace, + ] + subprocess.run(cmd, check=True) + + def copy_from_pod(self, remote_file_path, local_file_path): + cmd = [ + "kubectl", + "cp", + f"{self.pod_name}:{remote_file_path}", + local_file_path, + "-n", + self.namespace, + ] + subprocess.run(cmd, check=True) + + def upload_to_s3(self, file_path, bucket_name, object_name): + s3_client = boto3.client("s3") + s3_client.upload_file(file_path, bucket_name, object_name) + print(f"Uploaded {file_path} to s3://{bucket_name}/{object_name}") + + def cleanup_remote(self, remote_file_path): + cmd = [ + "kubectl", + "exec", + self.pod_name, + "-n", + self.namespace, + "--", + "rm", + "-rf", + remote_file_path, + ] + subprocess.run(cmd, check=True) + + def upload_to_gcs(self, file_path, bucket_name, object_name): + pass + + # client = gcs_storage.Client() + # bucket = client.bucket(bucket_name) + # blob = bucket.blob(object_name) + # blob.upload_from_filename(file_path) + # print(f"Uploaded {file_path} to gs://{bucket_name}/{object_name}") + + def upload_to_azure(self, file_path, connection_string, container_name, blob_name): + pass + + # blob_service_client = BlobServiceClient.from_connection_string(connection_string) + # blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name) + # with open(file_path, "rb") as data: + # blob_client.upload_blob(data) + # print(f"Uploaded {file_path} to azure://{container_name}/{blob_name}") + + def process_snapshot(self, remote_path, storage_type, storage_config): + remote_file_path = self.compress_remote_path(remote_path) + tar_file_path = os.path.basename(remote_file_path) + try: + self.copy_from_pod(remote_file_path, tar_file_path) + destination_path = f"{storage_config['backup_prefix']}/{os.path.basename(remote_file_path)}" + + if storage_type == "s3": + self.upload_to_s3( + tar_file_path, storage_config["bucket_name"], destination_path + ) + # elif storage_type == 'gcs': + # self.upload_to_gcs(tar_file_path, storage_config['bucket_name'], destination_path) + # elif storage_type == 'azure': + # self.upload_to_azure(tar_file_path, storage_config['connection_string'], storage_config['container_name'], destination_path) + else: + raise ValueError( + "Unsupported storage type. Choose 's3', 'gcs', or 'azure'." + ) + finally: + if os.path.exists(tar_file_path): + os.remove(tar_file_path) + self.cleanup_remote(remote_file_path) + + +# Example usage: +# if __name__ == "__main__": +# uploader = SnapshotUploader( +# pod_name='my-pod', # Replace with your pod name +# namespace='default' # Replace with your namespace +# ) + +# file_path = '/path/to/local/directory_or_file' +# remote_path = '/path/in/pod/directory_or_file.tar.gz' +# storage_type = 's3' # or 'gcs' or 'azure' +# storage_config = { +# 'bucket_name': 'your-s3-bucket-name', +# 'backup_prefix': 'your-s3-backup-prefix' +# } +# # For GCS +# # storage_config = { +# # 'bucket_name': 'your-gcs-bucket-name', +# # 'backup_prefix': 'your-gcs-backup-prefix' +# # } +# # For Azure +# # storage_config = { +# # 'connection_string': 'your-azure-connection-string', +# # 'container_name': 'your-container-name', +# # 'backup_prefix': 'your-azure-backup-prefix' +# # } + +# uploader.process_snapshot(file_path, remote_path, storage_type, storage_config) diff --git a/command-service/src/service/db_service.py b/command-service/src/service/db_service.py new file mode 100644 index 00000000..97aeb313 --- /dev/null +++ b/command-service/src/service/db_service.py @@ -0,0 +1,67 @@ +from typing import Callable + +import psycopg2 +import psycopg2.extras +from tenacity import retry, stop_after_attempt, wait_exponential + +from config import Config + + +def reconnect(func: Callable): + def wrapper(db_connection, *args, **kwargs): + tdecorator = retry(wait=wait_exponential(), stop=stop_after_attempt(3)) + decorated = tdecorator(func) + return decorated(db_connection, *args, **kwargs) + + return wrapper + + +class DatabaseService: + + def __init__(self): + self.config = Config() + + def connect(self): + # print("Connecting to postgresql db...") + db_host = self.config.find("postgres.db_host") + db_port = self.config.find("postgres.db_port") + db_user = self.config.find("postgres.db_user") + db_password = self.config.find("postgres.db_password") + database = self.config.find("postgres.database") + db_connection = psycopg2.connect( + database=database, + host=db_host, + port=db_port, + user=db_user, + password=db_password, + ) + db_connection.autocommit = True + return db_connection + + # @reconnect + def execute_select_one(self, sql): + db_connection = self.connect() + cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(sql) + result = cursor.fetchone() + db_connection.close() + return result + + # @reconnect + def execute_select_all(self, sql): + db_connection = self.connect() + cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(sql) + result = cursor.fetchall() + db_connection.close() + return result + + # @reconnect + def execute_upsert(self, sql): + db_connection = self.connect() + cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(sql) + record_count = cursor.rowcount + db_connection.close() + # print(f"{record_count} inserted/updated successfully") + return record_count diff --git a/command-service/src/service/detect_pii_service.py b/command-service/src/service/detect_pii_service.py new file mode 100644 index 00000000..b827b624 --- /dev/null +++ b/command-service/src/service/detect_pii_service.py @@ -0,0 +1,33 @@ +from typing import List + +from model.data_models import PIIError, PIIResult +from service.re_pii_model import REPIIModel + + +class DetectPIIService: + def __init__(self) -> None: + self.model = REPIIModel() + + def detect_pii_fields(self, event_data: dict) -> List[PIIResult] | PIIError: + try: + results = [] + for field in list(event_data.keys()): + results += self.model.detect_pii( + "address", field, str(event_data[field]) + ) + results += self.model.detect_pii( + "financial", field, str(event_data[field]) + ) + results += self.model.detect_pii("id", field, str(event_data[field])) + results += self.model.detect_pii( + "internet", field, str(event_data[field]) + ) + results += self.model.detect_pii("phone", field, str(event_data[field])) + return results + except Exception as err: + pii_error: PIIError = { + "errorCode": 500, + "errorMsg": type(err), + "errorTrace": err.args, + } + return pii_error diff --git a/command-service/src/service/http_service.py b/command-service/src/service/http_service.py index 16059f2e..9d136631 100644 --- a/command-service/src/service/http_service.py +++ b/command-service/src/service/http_service.py @@ -1,28 +1,33 @@ -import urllib3 -from urllib3.exceptions import NewConnectionError, MaxRetryError import backoff +import urllib3 +from urllib3.exceptions import MaxRetryError, NewConnectionError from model.data_models import HttpResponse -class HttpService(): + +class HttpService: def __init__(self): self.http = urllib3.PoolManager(num_pools=5) - @backoff.on_exception(wait_gen=backoff.expo, - exception=(NewConnectionError, MaxRetryError), - max_tries=3) + @backoff.on_exception( + wait_gen=backoff.expo, + exception=(NewConnectionError, MaxRetryError), + max_tries=3, + ) def post(self, url: str, body=None, headers=None): - response = self.http.request('POST', url, body=body, headers=headers) - return HttpResponse(status = response.status, body = response.data.decode("utf-8")) - - @backoff.on_exception(wait_gen=backoff.expo, - exception=(NewConnectionError, MaxRetryError), - max_tries=3) + response = self.http.request("POST", url, body=body, headers=headers) + return HttpResponse(status=response.status, body=response.data.decode("utf-8")) + + @backoff.on_exception( + wait_gen=backoff.expo, + exception=(NewConnectionError, MaxRetryError), + max_tries=3, + ) def get(self, url: str): - response = self.http.request('GET', url) - return HttpResponse(status = response.status, body = response.data.decode("utf-8")) - + response = self.http.request("GET", url) + return HttpResponse(status=response.status, body=response.data.decode("utf-8")) + def delete(self, url: str): - response = self.http.request('DELETE', url) - return HttpResponse(status = response.status, body = response.data.decode("utf-8")) \ No newline at end of file + response = self.http.request("DELETE", url) + return HttpResponse(status=response.status, body=response.data.decode("utf-8")) diff --git a/command-service/src/service/prometheus_backup.py b/command-service/src/service/prometheus_backup.py new file mode 100644 index 00000000..08dd18c5 --- /dev/null +++ b/command-service/src/service/prometheus_backup.py @@ -0,0 +1,48 @@ +import requests + +from config import Config +from service.backup_service import BackupService + + +class PrometheusBackup: + def __init__(self) -> None: + self.config = Config() + self.prometheus_host = self.config.find("prometheus.host") + self.prometheus_endpoint = self.config.find("prometheus.endpoint") + self.prom_pod = self.config.find("prometheus.pod") + self.prom_ns = self.config.find("prometheus.namespace") + self.backup_provider = self.config.find("backups.provider") + self.backup_bucket = self.config.find("backups.bucket") + self.backup_prefix = self.config.find("prometheus.backup_prefix") + + def backup_prometheus(self) -> None: + snapshot_uploader = BackupService( + pod_name=self.prom_pod, namespace=self.prom_ns + ) + + # trigger prometheus snapshot + snapshot_url = f"{self.prometheus_host}{self.prometheus_endpoint}" + + response = requests.post(snapshot_url) + + if response.status_code != 200 or response.json()["status"] != "success": + raise Exception(f"Failed to trigger prometheus snapshot: {response.text}") + + snapshot_file_path = "/prometheus/snapshots" + snapshot_id = response.json()["data"]["name"] + + snapshot_uploader.process_snapshot( + remote_path=f"{snapshot_file_path}/{snapshot_id}", + storage_type=self.backup_provider, + storage_config={ + "bucket_name": self.backup_bucket, + "backup_prefix": self.backup_prefix, + }, + ) + + snapshot_uploader.cleanup_remote(f"{snapshot_file_path}/{snapshot_id}") + + +if __name__ == "__main__": + prometheus_backup = PrometheusBackup() + prometheus_backup.backup_prometheus() diff --git a/command-service/src/service/re_pii_model.py b/command-service/src/service/re_pii_model.py new file mode 100644 index 00000000..52031946 --- /dev/null +++ b/command-service/src/service/re_pii_model.py @@ -0,0 +1,64 @@ +import re +from typing import List + +import yaml + +from model.data_models import PIIModel, PIIReason, PIIResult + + +class REPIIModel(PIIModel): + def __init__(self): + def join(loader, node): + seq = loader.construct_sequence(node) + return "".join([str(i) for i in seq]) + + yaml.add_constructor("!join", join) + with open("config/pii_rules.yml", "r") as f: + self.pii_rules = yaml.load(f, Loader=yaml.FullLoader) + + def detect_pii(self, entity, field, value) -> List[PIIResult]: + results = [] + reasons = [] + reasons += self.detect_entity(entity, value) + reasons += self.detect_entity_in_fieldname(entity, field) + if len(reasons) != 0: + score = 1 / len(reasons) + result: PIIResult = { + "field": field, + "type": entity, + "score": score, + "reason": reasons, + } + results.append(result) + return results + + def detect_entity(self, entity, value) -> List[PIIReason]: + matches = [] + for rule in list(self.pii_rules["values"][entity].keys()): + rule_matches = list( + re.findall(self.pii_rules["values"][entity][rule]["rule"], value) + ) + if len(rule_matches) != 0: + reason: PIIReason = { + "code": self.pii_rules["values"][entity][rule]["code"], + "resourceKey": self.pii_rules["values"][entity][rule][ + "resourceKey" + ], + "region": self.pii_rules["values"][entity][rule]["locale"], + "score": 1 / len(rule_matches), + } + matches.append(reason) + return matches + + def detect_entity_in_fieldname(self, entity, value) -> List[PIIReason]: + matches = [] + rule_matches = list(re.findall(self.pii_rules["keys"][entity]["rule"], value)) + if len(rule_matches) != 0: + reason: PIIReason = { + "code": self.pii_rules["keys"][entity]["code"], + "resourceKey": self.pii_rules["keys"][entity]["resourceKey"], + "region": self.pii_rules["keys"][entity]["locale"], + "score": 1 / len(rule_matches), + } + matches.append(reason) + return matches diff --git a/command-service/src/service/telemetry_service.py b/command-service/src/service/telemetry_service.py new file mode 100644 index 00000000..ebc9033d --- /dev/null +++ b/command-service/src/service/telemetry_service.py @@ -0,0 +1,38 @@ +import json +import os + +from kafka import KafkaProducer + +from config import Config +from model.telemetry_models import Actor, Audit, Context, Object, Pdata, Telemetry + + +class TelemetryService: + def __init__(self) -> None: + self.actor = Actor() + self.pdata = Pdata("{}.command.service".format(os.getenv("system_env", "dev"))) + self.context = Context(self.pdata, os.getenv("system_env", "dev")) + self.config_obj = Config() + try: + self.producer = KafkaProducer( + value_serializer=lambda v: json.dumps(v).encode("utf-8"), + bootstrap_servers=self.config_obj.find("kafka.brokers"), + ) + except Exception as e: + print("Error while connecting to kafka - ", e) + self.producer = None + self.topic = ( + os.getenv("system_env", "dev") + + "." + + self.config_obj.find("kafka.telemetry.topic") + ) + + def audit(self, object_: Object, edata: Audit): + event = Telemetry( + actor=self.actor, context=self.context, object=object_, edata=edata + ) + data = event.to_json() + print("Audit event - ", json.dumps(data, indent=4)) + + if self.producer: + self.producer.send(self.topic, data) diff --git a/command-service/src/stubs/snippets.py b/command-service/src/stubs/snippets.py new file mode 100644 index 00000000..89ffbbbd --- /dev/null +++ b/command-service/src/stubs/snippets.py @@ -0,0 +1,13 @@ +# Suspending a job in kubernetes +# def _suspend_job(self, job_name, namespace, results): +# result_status = False +# patch_cmd = """kubectl patch job/{job_name} --type=strategic --patch '{{"spec":{{"suspend":true}}}}' -n {namespace}""" +# patch_cmd = patch_cmd.format(job_name=job_name, namespace=namespace) +# patch_cmd_result = subprocess.run(patch_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True,) +# if patch_cmd_result.returncode == 0: +# print(f"Job {job_name} suspension succeeded...") +# result_status = True +# else: +# print(f"Error suspending job {job_name}: {patch_cmd_result.stderr.decode()}") +# results.append(result_status) +# return results From 0b5be36649170a2bea9cf5100dc11516f59d6b91 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 5 Jul 2024 18:00:17 +0530 Subject: [PATCH 012/235] #OBS-I116: Dataset CRUD APIs test and fixes --- .../DatasetCreateValidationSchema.json | 4 +-- .../v2/controllers/DatasetList/DatasetList.ts | 6 ++-- .../v2/controllers/DatasetRead/DatasetRead.ts | 4 +-- .../DatasetUpdate/DatasetUpdate.ts | 29 +++++-------------- .../DatasetUpdateValidationSchema.json | 6 ++-- api-service/src/v2/models/Dataset.ts | 3 +- api-service/src/v2/services/DatasetService.ts | 6 ++-- 7 files changed, 23 insertions(+), 35 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index 55cde742..938fca6d 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -260,10 +260,10 @@ }, "oneOf": [ { - "required": ["denorm_dataset", "denorm_out_field", "denorm_key"] + "required": ["dataset_id", "denorm_out_field", "denorm_key"] }, { - "required": ["denorm_dataset", "denorm_out_field", "jsonata_expr"] + "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] } ], "additionalProperties": false diff --git a/api-service/src/v2/controllers/DatasetList/DatasetList.ts b/api-service/src/v2/controllers/DatasetList/DatasetList.ts index 3003fc33..f73854ca 100644 --- a/api-service/src/v2/controllers/DatasetList/DatasetList.ts +++ b/api-service/src/v2/controllers/DatasetList/DatasetList.ts @@ -43,9 +43,9 @@ const listDatasets = async (request: Record): Promise { } as ErrorObject, req, res); return false; } - const fieldValues = _.split(fields, ",") + const fieldValues = fields ? _.split(fields, ",") : [] const invalidFields = _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes())) if (!_.isEmpty(invalidFields)) { logger.error({ code: "DATASET_INVALID_FIELDS", apiId, dataset_id, message: `The specified fields [${invalidFields}] in the dataset cannot be found` }) @@ -49,7 +49,7 @@ const datasetRead = async (req: Request, res: Response) => { } const { dataset_id } = req.params; const { fields, mode } = req.query; - const attributes = (fields === null) ? defaultFields : _.split(fields, ","); + const attributes = !fields ? defaultFields : _.split(fields, ","); const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes) if(!dataset) { logger.error({ code: "DATASET_NOT_FOUND", apiId, dataset_id, message: `Dataset with the given dataset_id:${dataset_id} not found` }) diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts index e7aa73c7..f5e68db4 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts @@ -43,29 +43,15 @@ const isValidRequest = async (req: Request, res: Response): Promise => return false; } - const duplicateDenormKeys = datasetService.getDuplicateDenormKey(_.get(req, ["body", "request", "denorm_config"])) - if (!_.isEmpty(duplicateDenormKeys)) { - const code = "DATASET_DUPLICATE_DENORM_KEY" - logger.error({ code: "DATASET_DUPLICATE_DENORM_KEY", body: req.body, message: `Duplicate denorm output fields found. Duplicate Denorm out fields are [${duplicateDenormKeys}]` }) - ResponseHandler.errorResponse({ - code: "DATASET_DUPLICATE_DENORM_KEY", - statusCode: 400, - message: "Duplicate denorm key found", - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - return false; - } - return true; } -const isValidDataset = (datasetModel: Model | null, req: Request, res: Response) : boolean => { +const isValidDataset = (dataset: Record | null, req: Request, res: Response): boolean => { const datasetId = _.get(req, ["body", "request", "dataset_id"]) const versionKey = _.get(req, ["body", "request", "version_key"]) - if(datasetModel) { - const dataset:Record = datasetModel.toJSON - if(dataset.api_version !== "v2") { + if (dataset) { + if (dataset.api_version !== "v2") { logger.error({ code: "DATASET_API_VERSION_MISMATCH", apiId, body: req.body, message: `Draft dataset api version is not v2:${datasetId}` }) ResponseHandler.errorResponse({ code: "DATASET_API_VERSION_MISMATCH", @@ -122,15 +108,16 @@ const datasetUpdate = async (req: Request, res: Response) => { } const draftDataset = mergeDraftDataset(datasetModel, datasetReq); - const response = datasetService.updateDraftDataset(draftDataset); + const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { - let dataset:Record = { - version_key: datasetReq.version_key, - name: datasetReq.name || _.get(datasetModel, ["name"]) + let dataset: Record = { + version_key: Date.now().toString(), + name: datasetReq.name || _.get(datasetModel, ["name"]), + id: _.get(datasetModel, ["id"]) } if(datasetReq.validation_config) dataset["validation_config"] = datasetReq.validation_config if(datasetReq.extraction_config) dataset["extraction_config"] = datasetReq.extraction_config diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 630cfbcc..929ac0bb 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -297,7 +297,7 @@ "if": { "properties": { "action": { - "enum": ["upsert", "update"] + "const": "upsert" } } }, @@ -306,10 +306,10 @@ "value": { "oneOf": [ { - "required": ["denorm_dataset", "denorm_out_field", "denorm_key"] + "required": ["dataset_id", "denorm_out_field", "denorm_key"] }, { - "required": ["denorm_dataset", "denorm_out_field", "jsonata_expr"] + "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] } ] } diff --git a/api-service/src/v2/models/Dataset.ts b/api-service/src/v2/models/Dataset.ts index 591bdc01..2989f309 100644 --- a/api-service/src/v2/models/Dataset.ts +++ b/api-service/src/v2/models/Dataset.ts @@ -80,6 +80,5 @@ export const Dataset = sequelize.define("datasets", { tableName: "datasets", timestamps: true, createdAt: "created_date", - updatedAt: "updated_date", - paranoid: true + updatedAt: "updated_date" }) \ No newline at end of file diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 9d8a613c..68d1da27 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -119,7 +119,8 @@ class DatasetService { await DatasetDraft.update(draftDataset, { where: { id: datasetId }, transaction}); await DatasetTransformationsDraft.destroy({ where: { dataset_id: datasetId }, transaction }); await DatasetSourceConfigDraft.destroy({ where: { dataset_id: datasetId }, transaction }); - } catch(err) { + await transaction.commit(); + } catch (err) { await transaction.rollback(); throw err; } @@ -140,7 +141,7 @@ class DatasetService { createDraftDatasetFromLive = async (dataset: Model) => { - let draftDataset:any = _.omit(dataset.toJSON, ["created_date", "updated_date", "published_date"]); + let draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); const dataset_config:any = _.get(dataset, "dataset_config"); const api_version:any = _.get(dataset, "api_version"); if(api_version === "v1") { @@ -174,6 +175,7 @@ class DatasetService { const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "datatype", "category"]); draftDataset["transformations_config"] = transformations } + draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version await DatasetDraft.create(draftDataset); return await this.getDraftDataset(draftDataset.dataset_id); From dbb1b9eb56eabdd0fd8db1f99df53d6f3313ead4 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Fri, 5 Jul 2024 18:15:46 +0530 Subject: [PATCH 013/235] #OBS-I115: cmd api remove addn modules --- command-service/requirements.txt | 1 - command-service/src/routes.py | 23 --- command-service/src/service/backup_service.py | 152 ------------------ .../src/service/prometheus_backup.py | 48 ------ 4 files changed, 224 deletions(-) delete mode 100644 command-service/src/service/backup_service.py delete mode 100644 command-service/src/service/prometheus_backup.py diff --git a/command-service/requirements.txt b/command-service/requirements.txt index 39b67b5d..6136ac6d 100644 --- a/command-service/requirements.txt +++ b/command-service/requirements.txt @@ -16,5 +16,4 @@ tenacity==8.2.2 kafka-python-ng==2.2.2 boto3 prometheus-client -kubernetes requests \ No newline at end of file diff --git a/command-service/src/routes.py b/command-service/src/routes.py index 7ee2839d..d94ce922 100644 --- a/command-service/src/routes.py +++ b/command-service/src/routes.py @@ -24,7 +24,6 @@ Result, ) from service.detect_pii_service import DetectPIIService -from service.prometheus_backup import PrometheusBackup app = FastAPI() command_executor = CommandExecutor() @@ -228,28 +227,6 @@ async def register_connector(req: FastAPIRequest): ) -prom_backup_endpoint = "/system/v1/backup/prometheus" - - -@app.get(prom_backup_endpoint) -def backup_prometheus() -> PlainTextResponse: - helper.onRequest( - entity="prometheus_backup", - id=None, - endpoint=prom_backup_endpoint, - dataset_id=None, - ) - prometheus_backup = PrometheusBackup() - prometheus_backup.backup_prometheus() - helper.onSuccessRequest( - entity="prometheus_backup", - id=None, - endpoint=prom_backup_endpoint, - dataset_id=None, - ) - return "Prometheus backup successful" - - if __name__ == "__main__": import uvicorn diff --git a/command-service/src/service/backup_service.py b/command-service/src/service/backup_service.py deleted file mode 100644 index f7fb8b54..00000000 --- a/command-service/src/service/backup_service.py +++ /dev/null @@ -1,152 +0,0 @@ -import os -import subprocess -import tarfile -import time - -import boto3 - -# from google.cloud import storage as gcs_storage -# from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient - - -class BackupService: - def __init__(self, pod_name, namespace="default"): - self.pod_name = pod_name - self.namespace = namespace - - def compress_file(self, file_path): - timestamp = time.strftime("%Y%m%d%H%M%S") - tar_file_path = f"{file_path}_{timestamp}.tar.gz" - with tarfile.open(tar_file_path, "w:gz") as tar: - tar.add(file_path, arcname=os.path.basename(file_path)) - return tar_file_path - - def compress_remote_path(self, remote_path): - timestamp = time.strftime("%Y%m%d%H%M%S") - tar_file_path = f"{remote_path}_{timestamp}.tar.gz" - compress_command = f"tar -czvf {tar_file_path} {os.path.dirname(remote_path)}/{os.path.basename(remote_path)}" - cmd = [ - "kubectl", - "exec", - self.pod_name, - "-n", - self.namespace, - "--", - "sh", - "-c", - compress_command, - ] - subprocess.run(cmd, check=True) - return tar_file_path - - def copy_to_pod(self, local_file_path, remote_file_path): - cmd = [ - "kubectl", - "cp", - local_file_path, - f"{self.pod_name}:{remote_file_path}", - "-n", - self.namespace, - ] - subprocess.run(cmd, check=True) - - def copy_from_pod(self, remote_file_path, local_file_path): - cmd = [ - "kubectl", - "cp", - f"{self.pod_name}:{remote_file_path}", - local_file_path, - "-n", - self.namespace, - ] - subprocess.run(cmd, check=True) - - def upload_to_s3(self, file_path, bucket_name, object_name): - s3_client = boto3.client("s3") - s3_client.upload_file(file_path, bucket_name, object_name) - print(f"Uploaded {file_path} to s3://{bucket_name}/{object_name}") - - def cleanup_remote(self, remote_file_path): - cmd = [ - "kubectl", - "exec", - self.pod_name, - "-n", - self.namespace, - "--", - "rm", - "-rf", - remote_file_path, - ] - subprocess.run(cmd, check=True) - - def upload_to_gcs(self, file_path, bucket_name, object_name): - pass - - # client = gcs_storage.Client() - # bucket = client.bucket(bucket_name) - # blob = bucket.blob(object_name) - # blob.upload_from_filename(file_path) - # print(f"Uploaded {file_path} to gs://{bucket_name}/{object_name}") - - def upload_to_azure(self, file_path, connection_string, container_name, blob_name): - pass - - # blob_service_client = BlobServiceClient.from_connection_string(connection_string) - # blob_client = blob_service_client.get_blob_client(container=container_name, blob=blob_name) - # with open(file_path, "rb") as data: - # blob_client.upload_blob(data) - # print(f"Uploaded {file_path} to azure://{container_name}/{blob_name}") - - def process_snapshot(self, remote_path, storage_type, storage_config): - remote_file_path = self.compress_remote_path(remote_path) - tar_file_path = os.path.basename(remote_file_path) - try: - self.copy_from_pod(remote_file_path, tar_file_path) - destination_path = f"{storage_config['backup_prefix']}/{os.path.basename(remote_file_path)}" - - if storage_type == "s3": - self.upload_to_s3( - tar_file_path, storage_config["bucket_name"], destination_path - ) - # elif storage_type == 'gcs': - # self.upload_to_gcs(tar_file_path, storage_config['bucket_name'], destination_path) - # elif storage_type == 'azure': - # self.upload_to_azure(tar_file_path, storage_config['connection_string'], storage_config['container_name'], destination_path) - else: - raise ValueError( - "Unsupported storage type. Choose 's3', 'gcs', or 'azure'." - ) - finally: - if os.path.exists(tar_file_path): - os.remove(tar_file_path) - self.cleanup_remote(remote_file_path) - - -# Example usage: -# if __name__ == "__main__": -# uploader = SnapshotUploader( -# pod_name='my-pod', # Replace with your pod name -# namespace='default' # Replace with your namespace -# ) - -# file_path = '/path/to/local/directory_or_file' -# remote_path = '/path/in/pod/directory_or_file.tar.gz' -# storage_type = 's3' # or 'gcs' or 'azure' -# storage_config = { -# 'bucket_name': 'your-s3-bucket-name', -# 'backup_prefix': 'your-s3-backup-prefix' -# } -# # For GCS -# # storage_config = { -# # 'bucket_name': 'your-gcs-bucket-name', -# # 'backup_prefix': 'your-gcs-backup-prefix' -# # } -# # For Azure -# # storage_config = { -# # 'connection_string': 'your-azure-connection-string', -# # 'container_name': 'your-container-name', -# # 'backup_prefix': 'your-azure-backup-prefix' -# # } - -# uploader.process_snapshot(file_path, remote_path, storage_type, storage_config) diff --git a/command-service/src/service/prometheus_backup.py b/command-service/src/service/prometheus_backup.py deleted file mode 100644 index 08dd18c5..00000000 --- a/command-service/src/service/prometheus_backup.py +++ /dev/null @@ -1,48 +0,0 @@ -import requests - -from config import Config -from service.backup_service import BackupService - - -class PrometheusBackup: - def __init__(self) -> None: - self.config = Config() - self.prometheus_host = self.config.find("prometheus.host") - self.prometheus_endpoint = self.config.find("prometheus.endpoint") - self.prom_pod = self.config.find("prometheus.pod") - self.prom_ns = self.config.find("prometheus.namespace") - self.backup_provider = self.config.find("backups.provider") - self.backup_bucket = self.config.find("backups.bucket") - self.backup_prefix = self.config.find("prometheus.backup_prefix") - - def backup_prometheus(self) -> None: - snapshot_uploader = BackupService( - pod_name=self.prom_pod, namespace=self.prom_ns - ) - - # trigger prometheus snapshot - snapshot_url = f"{self.prometheus_host}{self.prometheus_endpoint}" - - response = requests.post(snapshot_url) - - if response.status_code != 200 or response.json()["status"] != "success": - raise Exception(f"Failed to trigger prometheus snapshot: {response.text}") - - snapshot_file_path = "/prometheus/snapshots" - snapshot_id = response.json()["data"]["name"] - - snapshot_uploader.process_snapshot( - remote_path=f"{snapshot_file_path}/{snapshot_id}", - storage_type=self.backup_provider, - storage_config={ - "bucket_name": self.backup_bucket, - "backup_prefix": self.backup_prefix, - }, - ) - - snapshot_uploader.cleanup_remote(f"{snapshot_file_path}/{snapshot_id}") - - -if __name__ == "__main__": - prometheus_backup = PrometheusBackup() - prometheus_backup.backup_prometheus() From 94854f3cc87ed7da97adf986a131e6e998f7a27e Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 5 Jul 2024 18:29:55 +0530 Subject: [PATCH 014/235] #OBS-I116: Dataset status trasition to retire check for denorm fields --- .../DatasetStatusTransition.ts | 52 +++++++++++++------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 4fa47f85..1d710c76 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -164,22 +164,44 @@ const checkDatasetDenorm = async (payload: Record) => { const { dataset } = payload const { dataset_id, type } = dataset if (type === DatasetType.MasterDataset) { - const liveDatasets = await Dataset.findAll({ attributes: ["denorm_config"], raw: true }) || [] - const draftDatasets = await DatasetDraft.findAll({ attributes: ["denorm_config"], raw: true }) || [] - _.forEach([...liveDatasets, ...draftDatasets], datasets => { - _.forEach(_.get(datasets, "denorm_config.denorm_fields"), denorms => { - if (_.get(denorms, "dataset_id") === dataset_id) { - logger.error(`Failed to retire dataset as it is used by other datasets:${dataset_id}`) - throw { - code: "DATASET_IN_USE", - errCode: "BAD_REQUEST", - message: "Failed to retire dataset as it is used by other datasets", - statusCode: 400 - } - } - }) - }) + const liveDatasets = await Dataset.findAll({ where: { status: DatasetStatus.Live, type: DatasetType.Dataset }, attributes: ["denorm_config", "id"], raw: true }) || [] + const draftDatasets = await DatasetDraft.findAll({ where: { status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft], type: DatasetType.Dataset }, attributes: ["denorm_config", "id"], raw: true }) || [] + const liveDenorms = _.uniq(getDenormDatasets(liveDatasets, dataset_id)) + const draftDenorms = _.uniq(getDenormDatasets(draftDatasets, dataset_id)) + if (_.size([...liveDenorms, ...draftDenorms])) { + const denormErrMsg = getDenormErrMsg(liveDenorms, draftDenorms) + logger.error(denormErrMsg); + throw { + code: "DATASET_IN_USE", + errCode: "BAD_REQUEST", + message: denormErrMsg, + statusCode: 400 + } + } + } +} + +const getDenormErrMsg = (liveDenorms: Record, draftDenorms: Record) => { + if (_.size(liveDenorms) && _.size(draftDenorms)) { + return `Failed to retire dataset as it is used by Live datasets with id:[${liveDenorms}] and Draft datasets with id:[${draftDenorms}]` + } + if (_.size(liveDenorms)) { + return `Failed to retire dataset as it is used by Live datasets with id:[${liveDenorms}]` } + if (_.size(draftDenorms)) { + return `Failed to retire dataset as it is used by Draft datasets with id:[${draftDenorms}]` + } +} + +const getDenormDatasets = (datasets: any[], dataset_id: string) => { + const denorms = _.map(datasets, dataset => { + return _.map(_.get(dataset, "denorm_config.denorm_fields"), denorms => { + if (_.get(denorms, "dataset_id") === dataset_id) { + return _.get(dataset, "id") + } + }) + }) + return _.compact(_.flattenDeep(denorms)) } //SET_DATASET_TO_RETIRE From 876afad28ef469cf8e954c2984739fb1caf82cab Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sun, 7 Jul 2024 23:41:57 +0530 Subject: [PATCH 015/235] #OBS-I116: Dataset CRUD APIs test cases and fixes --- .../DatasetCreate/DatasetCreate.spec.ts | 73 +---- .../DatasetCreate/Fixtures.ts | 286 +++++++----------- .../DatasetList/DatasetList.spec.ts | 80 +---- .../DatasetManagement/DatasetList/Fixtures.ts | 192 ++---------- .../DatasetRead/DatasetRead.spec.ts | 110 ++----- .../DatasetManagement/DatasetRead/Fixtures.ts | 209 ++++--------- 6 files changed, 239 insertions(+), 711 deletions(-) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index 09b71821..0aec1b47 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -9,9 +9,6 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import { sequelize } from "../../../connections/databaseConnection"; import _ from "lodash"; import { apiId } from "../../../controllers/DatasetCreate/DatasetCreate" -import { DatasourceDraft } from "../../../models/DatasourceDraft"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; -import { DatasetTransformations } from "../../../models/Transformation"; import { Dataset } from "../../../models/Dataset"; chai.use(spies); @@ -31,32 +28,16 @@ describe("DATASET CREATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(null) + }) chai.spy.on(sequelize, "query", () => { return Promise.resolve([{ nextVal: 9 }]) }) - chai.spy.on(DatasourceDraft, "create", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: { id: "telemetry" } }) }) - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ "data_schema": {"$schema": "https://json-schema.org/draft/2020-12/schema","type": "object", - "properties": { - "eid": {"type": "string"}, - "ets": {"type": "string"} - }, - "additionalProperties": true - },}) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve() - }) - chai .request(app) .post("/v2/datasets/create") @@ -79,6 +60,9 @@ describe("DATASET CREATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(null) + }) chai .request(app) @@ -135,50 +119,5 @@ describe("DATASET CREATE API", () => { }); }); - it("Dataset creation failure: When timestamp key does not exist in the data schema", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve(null) - }) - chai.spy.on(sequelize, "query", () => { - return Promise.resolve([{ nextVal: 9 }]) - }) - chai.spy.on(DatasetDraft, "create", () => { - return Promise.resolve({ dataValues: { id: "telemetry" } }) - }) - chai - .request(app) - .post("/v2/datasets/create") - .send(TestInputsForDatasetCreate.DATASET_WITH_INVALID_TIMESTAMP) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Provided timestamp key not found in the data schema") - res.body.error.code.should.be.eq("DATASET_TIMESTAMP_NOT_FOUND") - done(); - }); - }); - - it("Dataset creation failure: Connection to the database failed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.reject({}) - }) - chai - .request(app) - .post("/v2/datasets/create") - .send(TestInputsForDatasetCreate.DATASET_WITH_DUPLICATE_DENORM_KEY) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to create dataset") - res.body.error.code.should.be.eq("DATASET_CREATION_FAILURE") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts index b2dbb7f3..b3138c9f 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -11,7 +11,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "validation_config": { "validate": true, @@ -44,49 +44,23 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "trip-details" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] - }, - "tags": [] - } - }, - - VALID_DATASET_WITH_DEFAULT_TS: { - "id": "api.datasets.create", - "ver": "v1", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "dataset_id": "sb-ddd", - "type": "dataset", - "name": "sb-telemetry2", - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - "additionalProperties": true - }, - "dataset_config": { - "data_key": "", - "timestamp_key": "obsrv_meta.syncts" + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, "tags": [] } @@ -101,7 +75,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -123,27 +97,19 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, - "transformations_config": [ - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - } - ], + "transformations_config": [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], "tags": [] } }, @@ -157,7 +123,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -179,42 +145,19 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] - }, - "transformations_config": [ - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - { - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - } - ], + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, + "transformations_config": [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, { "field_key": "ver", "transformation_function": { "type": "mask", "expr": "ver", "datatype": "string", "category": "pii" }, "mode": "Strict" }], "tags": [] } }, @@ -228,7 +171,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -250,9 +193,18 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, } }, @@ -265,7 +217,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "master-dataset", + "type": "master", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -287,9 +239,18 @@ export const TestInputsForDatasetCreate = { "additionalProperties": true }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, } }, VALID_MORE_THAN_MINIMAL_DATASET: { @@ -301,7 +262,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -325,14 +286,24 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "master-telemetry" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - } + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, } }, VALID_MORE_THAN_MINIMAL_MASTER_DATASET: { @@ -344,7 +315,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "master-dataset", + "type": "master", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -368,14 +339,24 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" - }, + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + } } }, VALID_MASTER_DATASET: { @@ -387,7 +368,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "master-dataset", + "type": "master", "name": "sb-telemetry2", "validation_config": { "validate": true, @@ -420,13 +401,23 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" } ] }, "dataset_config": { - "data_key": "", - "timestamp_key": "ets" + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true + }, + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] }, "tags": [] } @@ -444,41 +435,6 @@ export const TestInputsForDatasetCreate = { } }, - DATASET_WITH_INVALID_TIMESTAMP: { - "id": "api.datasets.create", - "ver": "v1", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "dataset_id": "sb-ddd", - "type": "dataset", - "name": "sb-telemetry2", - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "dataset_config": { - "data_key": "", - "timestamp_key": "lastAccessed" - }, - "tags": [] - } - }, - DATASET_WITH_DUPLICATE_DENORM_KEY: { "id": "api.datasets.create", "ver": "v2", @@ -488,7 +444,7 @@ export const TestInputsForDatasetCreate = { }, "request": { "dataset_id": "sb-ddd", - "type": "dataset", + "type": "event", "name": "sb-telemetry2", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", @@ -510,11 +466,13 @@ export const TestInputsForDatasetCreate = { "denorm_fields": [ { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" }, { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "telemetry" } ] } @@ -585,14 +543,7 @@ export const DATASET_CREATE_SUCCESS_FIXTURES = [ "httpStatus": httpStatus.OK, "status": "SUCCESS", "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - { - "title": "Dataset creation success: Geenerating ingestion spec successfully using the data schema", - "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_DEFAULT_TS, - "httpStatus": httpStatus.OK, - "status": "SUCCESS", - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, + } ] export const DATASET_FAILURE_DUPLICATE_DENORM_FIXTURES = [ @@ -602,12 +553,5 @@ export const DATASET_FAILURE_DUPLICATE_DENORM_FIXTURES = [ "httpStatus": httpStatus.BAD_REQUEST, "status": "FAILED", "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - { - "title": "Master Dataset creation failure: Dataset contains duplicate denorm out field", - "requestPayload": _.set(TestInputsForDatasetCreate.DATASET_WITH_DUPLICATE_DENORM_KEY, "request.type", "master-dataset"), - "httpStatus": httpStatus.BAD_REQUEST, - "status": "FAILED", - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" } ] \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts index 67344e5a..8b55ee75 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts @@ -5,12 +5,11 @@ import spies from "chai-spies"; import httpStatus from "http-status"; import { describe, it } from 'mocha'; import _ from "lodash"; -import { apiId, errorCode } from "../../../controllers/DatasetList/DatasetList"; +import { apiId } from "../../../controllers/DatasetList/DatasetList"; import { TestInputsForDatasetList } from "./Fixtures"; import { Dataset } from "../../../models/Dataset"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { DatasetTransformations } from "../../../models/Transformation"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; chai.use(spies); chai.should(); @@ -31,12 +30,6 @@ describe("DATASET LIST API", () => { chai.spy.on(DatasetDraft, "findAll", () => { return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) - }) chai .request(app) .post("/v2/datasets/list") @@ -50,7 +43,7 @@ describe("DATASET LIST API", () => { res.body.result.count.should.be.eq(2) res.body.params.msgid.should.be.eq(msgid) const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ..._.omit(TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA, ["data_version"]), version: 1, transformations_config: [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA, ["dataset_id"])] }, { ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA, "transformations_config": [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA, ["dataset_id"])] }]) + const expectedResult = JSON.stringify(TestInputsForDatasetList.VALID_RESPONSE) result.should.be.eq(expectedResult) done(); }); @@ -60,11 +53,8 @@ describe("DATASET LIST API", () => { chai.spy.on(DatasetDraft, "findAll", () => { return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([]) }) chai .request(app) @@ -79,7 +69,7 @@ describe("DATASET LIST API", () => { res.body.result.count.should.be.eq(1) res.body.params.msgid.should.be.eq(msgid) const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA, "transformations_config": [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA, ["dataset_id"])] }]) + const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA }]) result.should.be.eq(expectedResult) done(); }); @@ -89,11 +79,8 @@ describe("DATASET LIST API", () => { chai.spy.on(Dataset, "findAll", () => { return Promise.resolve([TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA]) }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) + chai.spy.on(DatasetDraft, "findAll", () => { + return Promise.resolve([]) }) chai .request(app) @@ -108,39 +95,7 @@ describe("DATASET LIST API", () => { res.body.result.count.should.be.eq(1) res.body.params.msgid.should.be.eq(msgid) const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ..._.omit(TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA, ["data_version"]), version: 1, transformations_config: [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA, ["dataset_id"])] }]) - result.should.be.eq(expectedResult) - done(); - }); - }); - - it("Dataset list success: When sortBy is provided in request payload", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA]) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA]) - }) - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.REQUEST_WITH_SORTBY) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.result.should.be.a("object") - res.body.result.count.should.be.eq(2) - res.body.params.msgid.should.be.eq(msgid) - const result = JSON.stringify(res.body.result.data) - const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_DRAFT_DATASET_SCHEMA, "transformations_config": [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_DRAFT_SCHEMA, ["dataset_id"])] }, { ..._.omit(TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA, ["data_version"]), version: 1, transformations_config: [_.omit(TestInputsForDatasetList.TRANSFORMATIONS_LIVE_SCHEMA, ["dataset_id"])] }]) + const expectedResult = JSON.stringify([{ ...TestInputsForDatasetList.VALID_LIVE_DATASET_SCHEMA}]) result.should.be.eq(expectedResult) done(); }); @@ -162,23 +117,4 @@ describe("DATASET LIST API", () => { }); }); - - it("Dataset list failure: Connection to the database failed", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.reject() - }) - chai - .request(app) - .post("/v2/datasets/list") - .send(TestInputsForDatasetList.REQUEST_WITHOUT_FILTERS) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) - res.body.error.message.should.be.eq("Failed to list dataset") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts index a905b1c0..31a5ba1c 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts @@ -1,181 +1,53 @@ export const TestInputsForDatasetList = { VALID_DRAFT_DATASET_SCHEMA: { "dataset_id": "telemetry", - "id": "telemetry.1", "name": "telemetry", - "type": "dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "msgid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] + "type": "events", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" ] }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, "tags": [ "tag1", "tag2" ], "status": "Draft", "version": 1, - "client_state": {}, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "2024-04-15 07:51:49.49", - "update_date": "", - "published_date": "" + "api_version": "v2" }, VALID_LIVE_DATASET_SCHEMA: { "dataset_id": "sb-telemetry", - "id": "sb-telemetry", "name": "sb-telemetry", - "type": "master-dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "msgid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] + "type": "master", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": true }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } + "keys_config": { + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" ] }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, "tags": [ "tag1", "tag2" ], "status": "Live", "data_version": 1, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "2024-04-16 07:51:49.49", - "update_date": "", - "published_date": "" - }, - TRANSFORMATIONS_DRAFT_SCHEMA: { - "dataset_id": "telemetry.1", - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - "section": "transformation" - } - }, - TRANSFORMATIONS_LIVE_SCHEMA: { - "dataset_id": "sb-telemetry", - "field_key": "eid", - "transformation_function": { - "type": "mask", - "expr": "eid", - "condition": null - }, - "mode": "Strict", - "metadata": { - "_transformationType": "mask", - "_transformedFieldDataType": "string", - "_transformedFieldSchemaType": "string", - } + "api_version": "v2" }, REQUEST_WITHOUT_FILTERS: { @@ -206,18 +78,7 @@ export const TestInputsForDatasetList = { "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, "request": { - "filters": { status: "Live", type: "master-dataset" } - } - }, - REQUEST_WITH_SORTBY: { - "id": "api.datasets.list", - "ver": "v2", - "ts": "2024-04-10T16:10:50+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" - }, - "request": { - "sortBy": [{ "field": "created_date", "order": "asc" }] + "filters": { status: "Live", type: "master" } } }, INVALID_REQUEST: { @@ -230,5 +91,6 @@ export const TestInputsForDatasetList = { "request": { "filters": { status: ["Ready"] } } - } + }, + VALID_RESPONSE: [{"dataset_id":"sb-telemetry","name":"sb-telemetry","type":"master","dataset_config":{"indexing_config":{"olap_store_enabled":false,"lakehouse_enabled":true,"cache_enabled":true},"keys_config":{"data_key":"ets"},"file_upload_path":["telemetry.json"]},"tags":["tag1","tag2"],"status":"Live","data_version":1,"api_version":"v2"},{"dataset_id":"telemetry","name":"telemetry","type":"events","dataset_config":{"indexing_config":{"olap_store_enabled":false,"lakehouse_enabled":true,"cache_enabled":false},"keys_config":{"timestamp_key":"ets"},"file_upload_path":["telemetry.json"]},"tags":["tag1","tag2"],"status":"Draft","version":1,"api_version":"v2"}] } \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index dce1b54c..97ec0282 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -8,13 +8,10 @@ import _ from "lodash"; import { apiId } from "../../../controllers/DatasetRead/DatasetRead"; import { TestInputsForDatasetRead } from "./Fixtures"; import { DatasetTransformations } from "../../../models/Transformation"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; import { Dataset } from "../../../models/Dataset"; import { DatasetDraft } from "../../../models/DatasetDraft"; -import { Datasource } from "../../../models/Datasource"; import { DatasetSourceConfig } from "../../../models/DatasetSourceConfig"; -import { DatasetSourceConfigDraft } from "../../../models/DatasetSourceConfigDraft"; -import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { ConnectorInstances } from "../../../models/ConnectorInstances"; chai.use(spies); chai.should(); @@ -27,8 +24,8 @@ describe("DATASET READ API", () => { }); it("Dataset read success: When minimal fields requested", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{ 'name': 'sb-telemetry', 'data_version': 1 }]) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ 'name': 'sb-telemetry', 'data_version': 1 }) }) chai .request(app) @@ -41,41 +38,35 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.name.should.be.eq('sb-telemetry') const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ 'name': 'sb-telemetry', 'version': 1 })) + result.should.be.eq(JSON.stringify({ name: 'sb-telemetry', data_version: 1 })) done(); }); }); it("Dataset read success: Fetch all dataset fields when fields param is empty", (done) => { - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.DRAFT_SCHEMA]) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.DRAFT_SCHEMA) }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?status=Draft") + .get("/v2/datasets/read/sb-telemetry?mode=edit") .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.type.should.be.eq('dataset') + res.body.result.type.should.be.eq('event') res.body.result.status.should.be.eq('Draft') const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA, "transformations_config": [] })) + result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA })) done(); }); }); it("Dataset read success: Fetch live dataset when status param is empty", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.LIVE_SCHEMA]) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.LIVE_SCHEMA) }) chai .request(app) @@ -88,12 +79,12 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.status.should.be.eq('Live') const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ..._.omit({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "transformations_config": TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA }, ["data_version"]), version: 1 })) + result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA })) done(); }); }); - it("Dataset read success: Creating draft on mode=edit if no draft found", (done) => { + it("Dataset read success: Creating draft on mode=edit if no draft found in v2", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve() }) @@ -103,30 +94,15 @@ describe("DATASET READ API", () => { chai.spy.on(DatasetTransformations, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) }) - chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.DATASOURCE_SCHEMA]) - }) - chai.spy.on(DatasetSourceConfig, "findAll", () => { + chai.spy.on(ConnectorInstances, "findAll", () => { return Promise.resolve([]) }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasourceDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfigDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) - }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?status=Draft&mode=edit") + .get("/v2/datasets/read/sb-telemetry?mode=edit") .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -135,41 +111,26 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.name.should.be.eq('sb-telemetry') const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA, "transformations_config": [] })) + result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) done(); }); }); - it("Dataset read success: Updating dataset status to draft on mode=edit if dataset status is Live", (done) => { + it("Dataset read success: Creating draft on mode=edit if no draft found in v1", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "sb-telemetry", name: "sb-telemetry", status: "Live", data_schema: {} }) + return Promise.resolve() }) chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "sb-telemetry", name: "sb-telemetry", status: "Live", data_version: 2, data_schema: {} }) + return Promise.resolve({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "api_version": "v1" }) }) chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) - }) - chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve([TestInputsForDatasetRead.DATASOURCE_SCHEMA]) + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) }) chai.spy.on(DatasetSourceConfig, "findAll", () => { return Promise.resolve([]) }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasourceDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfigDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) + chai.spy.on(DatasetDraft, "create", () => { + return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) }) chai .request(app) @@ -181,8 +142,8 @@ describe("DATASET READ API", () => { res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") res.body.result.name.should.be.eq('sb-telemetry') - const result = JSON.stringify(_.omit(res.body.result, "version_key")) - result.should.be.eq(JSON.stringify({"dataset_id":"sb-telemetry","name":"sb-telemetry","data_schema":{},"version":2,"status":"Draft","api_version":"v2","transformations_config":[]})) + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) done(); }); }); @@ -209,8 +170,8 @@ describe("DATASET READ API", () => { }); it("Dataset read failure: When the dataset of requested dataset_id not found", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([]) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve(null) }) chai .request(app) @@ -244,7 +205,7 @@ describe("DATASET READ API", () => { it("Dataset read failure: When specified field of draft dataset cannot be found", (done) => { chai .request(app) - .get("/v2/datasets/read/sb-telemetry?fields=data&status=Draft") + .get("/v2/datasets/read/sb-telemetry?fields=data&mode=edit") .end((err, res) => { res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") @@ -256,21 +217,4 @@ describe("DATASET READ API", () => { }); }); - it("Dataset read failure: Connection to the database failed", (done) => { - chai.spy.on(Dataset, "findAll", () => { - return Promise.reject() - }) - chai - .request(app) - .get("/v2/datasets/read/sb-telemetry") - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to read dataset") - res.body.error.code.should.be.eq("DATASET_READ_FAILURE") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts index 93c118e8..61760c6e 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -1,153 +1,56 @@ export const TestInputsForDatasetRead = { DRAFT_SCHEMA: { "dataset_id": "sb-telemetry", - "id": "sb-telemetry", "name": "sb-telemetry", - "type": "dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } - ] - }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, + "type": "event", + "status": "Draft", "tags": [ "tag1", "tag2" ], - "status": "Draft", "version": 1, - "client_state": {}, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "", - "update_date": "", - "published_date": "" + "api_version": "v2", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + } }, LIVE_SCHEMA: { - "dataset_id": "sb-telemetry", - "id": "sb-telemetry", "name": "sb-telemetry", - "type": "dataset", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 604800 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "eid": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "" - }, - "denorm_config": { - "redis_db_host": "localhost", - "redis_db_port": 6379, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - } - ] - }, - "dataset_config": { - "data_key": "eid", - "timestamp_key": "ets", - "entry_topic": "local.ingest", - "redis_db_host": "localhost", - "redis_db_port": 6379, - "index_data": true, - "redis_db": 0 - }, + "type": "event", + "status": "Live", "tags": [ "tag1", "tag2" ], - "status": "Live", "data_version": 1, - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "created_date": "", - "update_date": "", - "published_date": "" + "api_version": "v2", + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + } }, - TRANSFORMATIONS_SCHEMA: [ + TRANSFORMATIONS_SCHEMA:[{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], + TRANSFORMATIONS_SCHEMA_V1: [ { "field_key": "eid", "transformation_function": { @@ -164,27 +67,27 @@ export const TestInputsForDatasetRead = { } } ], - DATASOURCE_SCHEMA:{ - "id": "sb-telemetry_sb-telemetry", - "datasource": "sb-telemetry", - "dataset_id": "sb-telemetry", - "ingestion_spec": {"type":"kafka","spec":{"dataSchema":{"dataSource":"dataset-conf_day","dimensionsSpec":{"dimensions":[{"type":"string","name":"a"},{"type":"string","name":"obsrv.meta.source.connector"},{"type":"string","name":"obsrv.meta.source.id"}]},"timestampSpec":{"column":"obsrv_meta.syncts","format":"auto"},"metricsSpec":[],"granularitySpec":{"type":"uniform","segmentGranularity":"DAY","queryGranularity":"none","rollup":false}},"tuningConfig":{"type":"kafka","maxBytesInMemory":134217728,"maxRowsPerSegment":5000000,"logParseExceptions":true},"ioConfig":{"type":"kafka","consumerProperties":{"bootstrap.servers":"localhost:9092"},"taskCount":1,"replicas":1,"taskDuration":"PT1H","useEarliestOffset":true,"completionTimeout":"PT1H","inputFormat":{"type":"json","flattenSpec":{"useFieldDiscovery":true,"fields":[{"type":"path","expr":"$.['a']","name":"a"},{"type":"path","expr":"$.obsrv_meta.['syncts']","name":"obsrv_meta.syncts"},{"type":"path","expr":"$.obsrv_meta.source.['connector']","name":"obsrv.meta.source.connector"},{"type":"path","expr":"$.obsrv_meta.source.['connectorInstance']","name":"obsrv.meta.source.id"},{"expr":"$.obsrv_meta.syncts","name":"obsrv_meta.syncts","type":"path"}]}},"appendToExisting":false}}}, - "datasource_ref": "sb-telemetry_DAY", - "retention_period": { - "enabled": "false" - }, - "archival_policy": { - "enabled": "false" - }, - "purge_policy": { - "enabled": "false" - }, - "backup_config": { - "enabled": "false" - }, - "status": "Live", - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "published_date": "2023-07-03 00:00:00" + DATASOURCE_SCHEMA: { + "id": "sb-telemetry_sb-telemetry", + "datasource": "sb-telemetry", + "dataset_id": "sb-telemetry", + "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "dataset-conf_day", "dimensionsSpec": { "dimensions": [{ "type": "string", "name": "a" }, { "type": "string", "name": "obsrv.meta.source.connector" }, { "type": "string", "name": "obsrv.meta.source.id" }] }, "timestampSpec": { "column": "obsrv_meta.syncts", "format": "auto" }, "metricsSpec": [], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "none", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxBytesInMemory": 134217728, "maxRowsPerSegment": 5000000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "consumerProperties": { "bootstrap.servers": "localhost:9092" }, "taskCount": 1, "replicas": 1, "taskDuration": "PT1H", "useEarliestOffset": true, "completionTimeout": "PT1H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [{ "type": "path", "expr": "$.['a']", "name": "a" }, { "type": "path", "expr": "$.obsrv_meta.['syncts']", "name": "obsrv_meta.syncts" }, { "type": "path", "expr": "$.obsrv_meta.source.['connector']", "name": "obsrv.meta.source.connector" }, { "type": "path", "expr": "$.obsrv_meta.source.['connectorInstance']", "name": "obsrv.meta.source.id" }, { "expr": "$.obsrv_meta.syncts", "name": "obsrv_meta.syncts", "type": "path" }] } }, "appendToExisting": false } } }, + "datasource_ref": "sb-telemetry_DAY", + "retention_period": { + "enabled": "false" + }, + "archival_policy": { + "enabled": "false" + }, + "purge_policy": { + "enabled": "false" + }, + "backup_config": { + "enabled": "false" + }, + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "published_date": "2023-07-03 00:00:00" } } \ No newline at end of file From 5066e719f6cd1519b91dca0e5ac0d814212f7c05 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Mon, 8 Jul 2024 12:53:18 +0530 Subject: [PATCH 016/235] #OBS-I115: Dataset Transition API refactoring --- .../src/v2/configs/DatasetConfigDefault.ts | 4 +- .../DatasetStatusTransition.ts | 282 ++++++++++++------ .../ReadyToPublishSchema.json | 208 +++++++++---- api-service/src/v2/types/DatasetModels.ts | 2 +- 4 files changed, 340 insertions(+), 156 deletions(-) diff --git a/api-service/src/v2/configs/DatasetConfigDefault.ts b/api-service/src/v2/configs/DatasetConfigDefault.ts index 768a7ffd..376d9f54 100644 --- a/api-service/src/v2/configs/DatasetConfigDefault.ts +++ b/api-service/src/v2/configs/DatasetConfigDefault.ts @@ -42,8 +42,8 @@ export const defaultDatasetConfig = { "timestamp_key": ingestionConfig.indexCol["Event Arrival Time"] }, "cache_config": { - "redis_db_host": config.redis_config.dedup_redis_host, - "redis_db_port": config.redis_config.dedup_redis_port, + "redis_db_host": config.redis_config.denorm_redis_host, + "redis_db_port": config.redis_config.denorm_redis_port, "redis_db": 0 }, "file_upload_path": [] diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 4fa47f85..aa663cb1 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -19,16 +19,22 @@ import { Datasource } from "../../models/Datasource"; import { DatasetTransformations } from "../../models/Transformation"; import { executeCommand } from "../../connections/commandServiceConnection"; import { druidHttpService } from "../../connections/druidConnection"; +import { query, sequelize } from "../../connections/databaseConnection"; +import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; -export const apiId = "api.datasets.status-transition"; -export const errorCode = "DATASET_STATUS_TRANSITION_FAILURE" +const transitionFailed = "DATASET_STATUS_TRANSITION_FAILURE" +const invalidRequest = "DATASET_STATUS_TRANSITION_INVALID_INPUT" +const datasetNotFound = "DATASET_NOT_FOUND" -const allowedTransitions = { +const allowedTransitions: Record = { Delete: [DatasetStatus.Draft, DatasetStatus.ReadyToPublish], ReadyToPublish: [DatasetStatus.Draft], Live: [DatasetStatus.ReadyToPublish], Retire: [DatasetStatus.Live], + Archive: [DatasetStatus.Retired], + Purge: [DatasetStatus.Archived] } +const liveDatasetActions = ["Retire", "Archive", "Purge"] const statusTransitionCommands = { Delete: ["DELETE_DRAFT_DATASETS"], @@ -37,95 +43,120 @@ const statusTransitionCommands = { Retire: ["CHECK_DATASET_IS_DENORM", "SET_DATASET_TO_RETIRE", "DELETE_SUPERVISORS", "RESTART_PIPELINE"] } -const datasetStatusTransition = async (req: Request, res: Response) => { - const requestBody = req.body - const msgid = _.get(req, ["body", "params", "msgid"]); - const resmsgid = _.get(res, "resmsgid"); - try { - const { dataset_id, status } = _.get(requestBody, "request"); - - const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) - if (!isRequestValid.isValid) { - const code = "DATASET_STATUS_TRANSITION_INVALID_INPUT" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } +const logHeaders = (req: Request, res: Response) => { + return { + apiId: "api.datasets.status-transition", msgid:_.get(req, ["body", "params", "msgid"]), request: req.body, resmsgid: _.get(res, "resmsgid") + } +} - const datasetRecord = await fetchDataset({ status, dataset_id }) - if (_.isEmpty(datasetRecord)) { - const code = "DATASET_NOT_FOUND" - const errorMessage = getErrorMessage(status, code) - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `${errorMessage} for dataset:${dataset_id}` }) - return ResponseHandler.errorResponse({ - code, - message: errorMessage, - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); - } +const validateRequest = (req: Request, res: Response, headers: Record): boolean => { + const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) + if (!isRequestValid.isValid) { + logger.error({ code: invalidRequest, headers, message: isRequestValid.message }) + ResponseHandler.errorResponse({ + code: invalidRequest, + message: isRequestValid.message, + statusCode: 400, + errCode: "BAD_REQUEST" + } as ErrorObject, req, res); + return false; + } + return true; +} - const allowedStatus = _.get(allowedTransitions, status) - const datasetStatus = _.get(datasetRecord, "status") - if (!_.includes(allowedStatus, datasetStatus)) { - const code = `DATASET_${_.toUpper(status)}_FAILURE` - const errorMessage = getErrorMessage(status, "STATUS_INVALID") - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: `${errorMessage} for dataset:${dataset_id} status:${datasetStatus} with status transition to ${status}` }) - return ResponseHandler.errorResponse({ - code, - message: errorMessage, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } +const validateDataset = (req: Request, res: Response, dataset: any, action: string, headers: Record) : boolean => { + + if (_.isEmpty(dataset)) { + logger.error({ code: datasetNotFound, headers, message: `Dataset not found for dataset:${dataset.id}` }) + ResponseHandler.errorResponse({ + code: datasetNotFound, + message: `Dataset not found for dataset: ${dataset.id}`, + statusCode: 404, + errCode: "NOT_FOUND" + } as ErrorObject, req, res); + return false; + } - const transitionCommands = _.get(statusTransitionCommands, status) - await executeTransition({ transitionCommands, dataset: datasetRecord }) - - logger.info({ apiId, msgid, requestBody, resmsgid, message: `Dataset status transition to ${status} successful with id:${dataset_id}` }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: `Dataset status transition to ${status} successful`, dataset_id } }); - } catch (error: any) { - const code = _.get(error, "code") || errorCode - logger.error(error, apiId, msgid, code, requestBody, resmsgid) - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code, message: "Failed to perform status transition on datasets" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } + if(!_.includes(allowedTransitions[action], dataset.status)) { + const code = `DATASET_${_.toUpper(action)}_FAILURE` + logger.error({ code, headers, message: `${errorMessage} for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}` }) + ResponseHandler.errorResponse({ + code: datasetNotFound, + message: `${errorMessage} for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, + statusCode: 404, + errCode: "NOT_FOUND" + } as ErrorObject, req, res); + return false; + } + + return true; } -const fetchDataset = async (configs: Record) => { - const { dataset_id, status } = configs - if (_.includes([DatasetAction.ReadyToPublish, DatasetAction.Delete], status)) { - return getDraftDatasetRecord(dataset_id) + +const datasetStatusTransition = async (req: Request, res: Response) => { + + const headers = logHeaders(req, res) + const { dataset_id, status } = _.get(req.body, "request"); + if (!validateRequest(req, res, headers)) { + return; } - if (_.includes([DatasetAction.Live], status)) { - return datasetService.getDraftDataset(dataset_id) + + const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status"]) + + if(!validateDataset(req, res, dataset, status, headers)) { + return; } - if (_.includes([DatasetAction.Retire], status)) { - return datasetService.getDataset(dataset_id) + + switch(status) { + case "Delete": + await deleteDataset(dataset); + break; + case "ReadyToPublish": + await readyForPublish(dataset); + break; + case "Live": + await publishDataset(dataset); + break; + case "Retire": + await retireDataset(dataset); + break; + case "Archive": + await archiveDataset(dataset); + break; + case "Purge": + await purgeDataset(dataset); + break; } + + logger.info({ headers, message: `Dataset status transition to ${status} successful with id:${dataset_id}` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: `Dataset status transition to ${status} successful`, dataset_id } }); + } -const executeTransition = async (configs: Record) => { - const { transitionCommands, dataset } = configs - const transitionPromises = _.map(transitionCommands, async command => { - const commandWorkflow = _.get(commandExecutors, command) - return commandWorkflow({ dataset }) - }) - await Promise.all(transitionPromises) + +// Delete a draft dataset +const deleteDataset = async (dataset: Record) => { + + // TODO: Delete any sample files or schemas that are uploaded + const { id } = dataset + const transaction = await sequelize.transaction() + try { + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) + await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) + await DatasetDraft.destroy({ where: { id } , transaction}) + await transaction.commit() + } catch (err) { + await transaction.rollback() + throw err + } } -//VALIDATE_DATASET_CONFIGS -const validateDataset = async (configs: Record) => { - const { dataset } = configs - const datasetValid: Record = schemaValidation(dataset, ReadyToPublishSchema) + +const readyForPublish = async (dataset: Record) => { + + const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + const datasetValid: Record = schemaValidation(draftDataset, ReadyToPublishSchema) if (!datasetValid.isValid) { throw { code: "DATASET_CONFIGS_INVALID", @@ -134,29 +165,81 @@ const validateDataset = async (configs: Record) => { statusCode: 400 } } - await DatasetDraft.update({ status: DatasetStatus.ReadyToPublish }, { where: { id: dataset.id } }) + _.set(draftDataset, 'status', DatasetStatus.ReadyToPublish) + await DatasetDraft.update(draftDataset, { where: { id: dataset.id } }) } -//DELETE_DRAFT_DATASETS -const deleteDataset = async (configs: Record) => { - const { dataset } = configs - const { id } = dataset - await deleteDraftRecords({ dataset_id: id }) + + +//PUBLISH_DATASET +const publishDataset = async (dataset: Record) => { + + const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + + await validateAndUpdateDenormConfig(draftDataset); + await updateMaterDataConfig(draftDataset) + await DatasetDraft.update(draftDataset, { where: { id: dataset.id } }) + await executeCommand(dataset.id, "PUBLISH_DATASET"); } -const deleteDraftRecords = async (config: Record) => { - const { dataset_id } = config; - await DatasetTransformationsDraft.destroy({ where: { dataset_id } }) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id } }) - await DatasourceDraft.destroy({ where: { dataset_id } }) - await DatasetDraft.destroy({ where: { id: dataset_id } }) +const validateAndUpdateDenormConfig = async (draftDataset: any) => { + + // 1. Check if there are denorm fields and dependent master datasets are published + const denormConfig = _.get(draftDataset, "denorm_config") + if(denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { + const datasetIds = _.map(denormConfig.denorm_fields, 'dataset_id') + const masterDatasets = await datasetService.findDatasets({id: datasetIds, type: "master"}, ["id", "status", "dataset_config", "api_version"]) + const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { + const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) + let datasetStatus : Record = { + dataset_id: denormField.dataset_id, + exists: (md) ? true : false, + isLive: (md) ? md.status === "Live" : false, + status: md.status + } + if(md.api_version === "v2") + datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); + else + datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); + + return datasetStatus; + }) + const invalidMasters = _.filter(masterDatasetsStatus, {isLive: false}) + if(_.size(invalidMasters) > 0) { + const invalidIds = _.map(invalidMasters, 'dataset_id') + throw { + code: "DEPENDENT_MASTER_DATA_NOT_LIVE", + message: `The datasets with id:${invalidIds} are not in published status`, + errCode: "DEPENDENT_MASTER_DATA_NOT_LIVE", + statusCode: 428 + } + } + + // 2. Populate redis db for denorm + draftDataset["denorm_config"] = { + redis_db_host: defaultDatasetConfig.denorm_config.redis_db_host, + redis_db_port: defaultDatasetConfig.denorm_config.redis_db_port, + denorm_fields: _.map(masterDatasetsStatus, 'denorm_field') + } + } } -//PUBLISH_DATASET -const publishDataset = async (configs: Record) => { - const { dataset } = configs - const { dataset_id } = dataset - await executeCommand(dataset_id, "PUBLISH_DATASET"); +const updateMaterDataConfig = async (draftDataset: any) => { + if(draftDataset.type === 'master') { + if(draftDataset.dataset_config.cache_config.redis_db === 0) { + const { results }: any = await query("SELECT nextval('redis_db_index')") + if(_.isEmpty(results)) { + throw { + code: "REDIS_DB_INDEX_FETCH_FAILED", + message: `Unable to fetch the redis db index for the master data`, + errCode: "REDIS_DB_INDEX_FETCH_FAILED", + statusCode: 500 + } + } + const nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; + _.set(draftDataset, 'dataset_config.cache_config.redis_db', nextRedisDB) + } + } } //CHECK_DATASET_IS_DENORM @@ -166,6 +249,11 @@ const checkDatasetDenorm = async (payload: Record) => { if (type === DatasetType.MasterDataset) { const liveDatasets = await Dataset.findAll({ attributes: ["denorm_config"], raw: true }) || [] const draftDatasets = await DatasetDraft.findAll({ attributes: ["denorm_config"], raw: true }) || [] + + _.includes( + _.map(_.flatten(_.map(liveDatasets, "denorm_config.denorm_fields")), 'dataset_id'), + dataset_id + ) _.forEach([...liveDatasets, ...draftDatasets], datasets => { _.forEach(_.get(datasets, "denorm_config.denorm_fields"), denorms => { if (_.get(denorms, "dataset_id") === dataset_id) { diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index 743b2944..c5e7785e 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -186,11 +186,11 @@ "type": "string", "minLength": 1 }, - "dataset_name": { + "dataset_id": { "type": "string", "minLength": 1 }, - "dataset_id": { + "jsonata_expr": { "type": "string", "minLength": 1 }, @@ -198,16 +198,19 @@ "type": "integer" } }, - "required": [ - "denorm_key", - "denorm_out_field", - "dataset_id" + "oneOf": [ + { + "required": ["dataset_id", "denorm_out_field", "denorm_key"] + }, + { + "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] + } ], "additionalProperties": false } } }, - "required": ["redis_db_host", "redis_db_port", "denorm_fields"], + "required": ["denorm_fields"], "additionalProperties": false }, "router_config": { @@ -224,58 +227,74 @@ "dataset_config": { "type": "object", "properties": { - "mergedEvent": { - "type": "object" - }, - "configurations": { - "type": "object" - }, - "dataMappings": { - "type": "object" - }, - "data_key": { - "type": "string" - }, - "timestamp_key": { - "type": "string", - "minLength": 1 - }, - "entry_topic": { - "type": "string", - "minLength": 1 - }, - "redis_db_host": { - "type": "string", - "minLength": 1 - }, - "exclude_fields": { - "type": "array" - }, - "redis_db_port": { - "type": "integer" - }, - "index_data": { - "type": "boolean" - }, - "redis_db": { - "type": "integer" - }, "file_upload_path": { "type": "array", "items": { "type": "string", "minLength": 1 } + }, + "dataset_tz": { + "type": "string" + }, + "indexing_config": { + "type": "object", + "properties": { + "olap_store_enabled": { + "type": "boolean", + "default": false + }, + "lakehouse_enabled": { + "type": "boolean", + "default": true + }, + "cache_enabled": { + "type": "boolean", + "default": false + } + }, + "required": ["olap_store_enabled", "lakehouse_enabled", "cache_enabled"], + "additionalProperties": false + }, + "keys_config": { + "type": "object", + "properties": { + "data_key": { + "type": "string" + }, + "partition_key": { + "type": "string" + }, + "timestamp_key": { + "type": "string" + }, + "timestamp_format": { + "type": "string", + "minLength": 1 + } + }, + "required": ["data_key", "timestamp_key"], + "additionalProperties": false + }, + "cache_config": { + "type": "object", + "properties": { + "redis_db_host": { + "type": "string", + "minLength": 1 + }, + "redis_db_port": { + "type": "integer" + }, + "redis_db": { + "type": "integer" + } + }, + "required": ["redis_db_host", "redis_db_port"], + "additionalProperties": false } }, - "required": [ - "timestamp_key", - "entry_topic", - "redis_db_host", - "redis_db_port", - "redis_db", - "index_data" - ], + "required": ["indexing_config", "keys_config", "cache_config"], "additionalProperties": false }, "tags": { @@ -285,14 +304,89 @@ "minLength": 1 } }, - "client_state": { - "type": "object" + "sample_data": { + "type": "string" + }, + "transformations_config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "field_key": { + "type": "string", + "minLength": 1 + }, + "transformation_function": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + }, + "condition": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "expr"], + "additionalProperties": false + }, + "datatype": { + "type": "string" + }, + "category": { + "type": "string", + "enum": ["pii", "transform", "derived"] + } + }, + "required": ["type", "expr", "category"], + "additionalProperties": false + }, + "mode": { + "type": "string", + "enum": ["Strict", "Lenient"] + } + }, + "additionalProperties": false, + "required": ["field_key", "transformation_function", "mode"] + } + }, + "connectors_config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "connector_id": { + "type": "string", + "minLength": 1 + }, + "connector_config": { + "type": "object" + }, + "operations_config": { + "type": "object" + } + }, + "additionalProperties": false, + "required": ["connector_id", "connector_config"] + } } }, "if": { "properties": { "type": { - "const": "master-dataset" + "const": "master" } } }, @@ -312,6 +406,7 @@ "dataset_id", "type", "name", + "version", "data_schema", "id", "validation_config", @@ -320,8 +415,9 @@ "denorm_config", "dedup_config", "router_config", - "client_state", "version_key", - "status" + "status", + "transformations_config", + "connectors_config" ] } diff --git a/api-service/src/v2/types/DatasetModels.ts b/api-service/src/v2/types/DatasetModels.ts index b22cb8b6..bef92f6e 100644 --- a/api-service/src/v2/types/DatasetModels.ts +++ b/api-service/src/v2/types/DatasetModels.ts @@ -45,7 +45,7 @@ export interface Result { } export enum DatasetStatus { - Live = "Live", Retired = "Retired", Draft = "Draft", ReadyToPublish = "ReadyToPublish" + Live = "Live", Retired = "Retired", Draft = "Draft", ReadyToPublish = "ReadyToPublish", Archived = "Archived" } export enum TransformationMode { From 32811a1d5c8617c389cfed0ed31768b9545f86b4 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Mon, 8 Jul 2024 13:45:11 +0530 Subject: [PATCH 017/235] #OBS-I115: Dataset Transition API refactoring --- .../DatasetStatusTransition.ts | 155 +++++++++--------- api-service/src/v2/types/DatasetModels.ts | 2 +- 2 files changed, 77 insertions(+), 80 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index e416ee4c..8512af8e 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -101,7 +101,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { return; } - const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status"]) + const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type"]) if(!validateDataset(req, res, dataset, status, headers)) { return; @@ -179,6 +179,8 @@ const publishDataset = async (dataset: Record) => { await validateAndUpdateDenormConfig(draftDataset); await updateMaterDataConfig(draftDataset) await DatasetDraft.update(draftDataset, { where: { id: dataset.id } }) + await generateDataSouce(draftDataset) + await executeCommand(dataset.id, "PUBLISH_DATASET"); } @@ -188,6 +190,14 @@ const validateAndUpdateDenormConfig = async (draftDataset: any) => { const denormConfig = _.get(draftDataset, "denorm_config") if(denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { const datasetIds = _.map(denormConfig.denorm_fields, 'dataset_id') + if(_.includes(datasetIds, draftDataset.id)) { + throw { + code: "SELF_REFERENCING_MASTER_DATA", + message: `The denorm master dataset is self-referencing itself`, + errCode: "SELF_REFERENCING_MASTER_DATA", + statusCode: 409 + } + } const masterDatasets = await datasetService.findDatasets({id: datasetIds, type: "master"}, ["id", "status", "dataset_config", "api_version"]) const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) @@ -242,68 +252,69 @@ const updateMaterDataConfig = async (draftDataset: any) => { } } -//CHECK_DATASET_IS_DENORM -const checkDatasetDenorm = async (payload: Record) => { - const { dataset } = payload - const { dataset_id, type } = dataset - if (type === DatasetType.MasterDataset) { - const liveDatasets = await Dataset.findAll({ where: { status: DatasetStatus.Live, type: DatasetType.Dataset }, attributes: ["denorm_config", "id"], raw: true }) || [] - const draftDatasets = await DatasetDraft.findAll({ where: { status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft], type: DatasetType.Dataset }, attributes: ["denorm_config", "id"], raw: true }) || [] - const liveDenorms = _.uniq(getDenormDatasets(liveDatasets, dataset_id)) - const draftDenorms = _.uniq(getDenormDatasets(draftDatasets, dataset_id)) - if (_.size([...liveDenorms, ...draftDenorms])) { - const denormErrMsg = getDenormErrMsg(liveDenorms, draftDenorms) +const generateDataSouce = async (draftDataset: any) => { + +} + +const retireDataset = async (dataset: Record) => { + + await canRetireIfMasterDataset(dataset); + await updateDatasetStatusToRetired(dataset); + await deleteDruidSupervisors(dataset); + await restartPipeline(dataset); +} + + +const canRetireIfMasterDataset = async (dataset: Record) => { + + if (dataset.type === DatasetType.master) { + + const liveDatasets = await Dataset.findAll({ where: { status: DatasetStatus.Live}, attributes: ["denorm_config", "id", "status"], raw: true }) || [] + const draftDatasets = await DatasetDraft.findAll({ where: { status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, attributes: ["denorm_config", "id", "status"], raw: true }) || [] + const allDatasets = _.union(liveDatasets, draftDatasets) + const extractDenormFields = _.map(allDatasets, function(depDataset) { + return {dataset_id: _.get(depDataset, 'id'), status: _.get(depDataset, 'status'), denorm_datasets: _.map(_.get(depDataset, 'denorm_config.denorm_fields'), 'dataset_id')} + }) + const deps = _.filter(extractDenormFields, function(depDS) { return _.includes(depDS.denorm_datasets, dataset.id)}) + if (_.size(deps) > 0) { + const denormErrMsg = `Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset` logger.error(denormErrMsg); throw { code: "DATASET_IN_USE", errCode: "BAD_REQUEST", message: denormErrMsg, - statusCode: 400 + statusCode: 400, + data: _.map(deps, function(o) { return _.omit(o, 'denorm_datasets')}) } } } } -const getDenormErrMsg = (liveDenorms: Record, draftDenorms: Record) => { - if (_.size(liveDenorms) && _.size(draftDenorms)) { - return `Failed to retire dataset as it is used by Live datasets with id:[${liveDenorms}] and Draft datasets with id:[${draftDenorms}]` - } - if (_.size(liveDenorms)) { - return `Failed to retire dataset as it is used by Live datasets with id:[${liveDenorms}]` - } - if (_.size(draftDenorms)) { - return `Failed to retire dataset as it is used by Draft datasets with id:[${draftDenorms}]` - } -} +const updateDatasetStatusToRetired = async (dataset: Record) => { -const getDenormDatasets = (datasets: any[], dataset_id: string) => { - const denorms = _.map(datasets, dataset => { - return _.map(_.get(dataset, "denorm_config.denorm_fields"), denorms => { - if (_.get(denorms, "dataset_id") === dataset_id) { - return _.get(dataset, "id") - } - }) - }) - return _.compact(_.flattenDeep(denorms)) + const transaction = await sequelize.transaction() + try { + await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }) + await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }) + await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}) + await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}) + await transaction.commit() + } catch(err:any) { + await transaction.rollback() + throw { + code: "UNABLE_TO_RETIRE_DATASET", + errCode: "SERVER_ERROR", + message: err.message, + statusCode: 500 + } + } } -//SET_DATASET_TO_RETIRE -const setDatasetRetired = async (config: Record) => { - const { dataset } = config; - const { dataset_id } = dataset - await Dataset.update({ status: DatasetStatus.Retired }, { where: { dataset_id } }) - await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id } }) - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id } }) - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id } }) -} +const deleteDruidSupervisors = async (dataset: Record) => { -//DELETE_SUPERVISORS -const deleteSupervisors = async (configs: Record) => { - const { dataset } = configs - const { type, dataset_id } = dataset try { - if (type !== DatasetType.MasterDataset) { - const datasourceRefs = await Datasource.findAll({ where: { dataset_id }, attributes: ["datasource_ref"], raw: true }) + if (dataset.type !== DatasetType.master) { + const datasourceRefs = await Datasource.findAll({ where: { dataset_id: dataset.id }, attributes: ["datasource_ref"], raw: true }) for (const sourceRefs of datasourceRefs) { const datasourceRef = _.get(sourceRefs, "datasource_ref") await druidHttpService.post(`/druid/indexer/v1/supervisor/${datasourceRef}/terminate`) @@ -311,47 +322,33 @@ const deleteSupervisors = async (configs: Record) => { } } } catch (error: any) { - logger.error({ error: _.get(error, "message"), message: `Failed to delete supervisors for dataset:${dataset_id}` }) + logger.error({ error: _.get(error, "message"), message: `Failed to delete supervisors for dataset:${dataset.id}` }) } } //RESTART_PIPELINE -const restartPipeline = async (config: Record) => { - const dataset_id = _.get(config, ["dataset", "dataset_id"]) - return executeCommand(dataset_id, "RESTART_PIPELINE") -} - -const commandExecutors = { - DELETE_DRAFT_DATASETS: deleteDataset, - PUBLISH_DATASET: publishDataset, - CHECK_DATASET_IS_DENORM: checkDatasetDenorm, - SET_DATASET_TO_RETIRE: setDatasetRetired, - DELETE_SUPERVISORS: deleteSupervisors, - RESTART_PIPELINE: restartPipeline, - VALIDATE_DATASET_CONFIGS: validateDataset +const restartPipeline = async (dataset: Record) => { + return executeCommand(dataset.id, "RESTART_PIPELINE") } -const getDraftDatasetRecord = async (dataset_id: string) => { - return DatasetDraft.findOne({ where: { id: dataset_id }, raw: true }); -} +const archiveDataset = async (dataset: Record) => { -const errorMessage = { - DATASET_NOT_FOUND: { - Delete: "Dataset not found to delete", - Retire: "Dataset not found to retire", - ReadyToPublish: "Dataset not found to perform status transition to ready to publish", - Live: "Dataset not found to perform status transition to live" - }, - STATUS_INVALID: { - Delete: "Failed to Delete dataset", - Retire: "Failed to Retire dataset as it is not in live state", - ReadyToPublish: "Failed to mark dataset Ready to publish as it not in draft state", - Live: "Failed to mark dataset Live as it is not in ready to publish state" + throw { + code: "ARCHIVE_NOT_IMPLEMENTED", + errCode: "NOT_IMPLEMENTED", + message: "Archive functionality is not implemented", + statusCode: 501 } } -const getErrorMessage = (status: string, code: string) => { - return _.get(errorMessage, [code, status]) || "Failed to perform status transition" +const purgeDataset = async (dataset: Record) => { + + throw { + code: "PURGE_NOT_IMPLEMENTED", + errCode: "NOT_IMPLEMENTED", + message: "Purge functionality is not implemented", + statusCode: 501 + } } export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/v2/types/DatasetModels.ts b/api-service/src/v2/types/DatasetModels.ts index bef92f6e..4de2a5b9 100644 --- a/api-service/src/v2/types/DatasetModels.ts +++ b/api-service/src/v2/types/DatasetModels.ts @@ -57,7 +57,7 @@ export enum ValidationMode { } export enum DatasetType { - Dataset = "dataset", MasterDataset = "master-dataset" + event = "event", master = "master", transaction = "transaction" } export enum DatasetAction { From 32d0bbe73e93683fbfa76cdd060df1af4bd54d7a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 10 Jul 2024 23:16:48 +0530 Subject: [PATCH 018/235] #OBS-I116: Dataset update API Dedupe and denorm test cases fixes --- .../DatasetStatusTransition.ts | 5 +- .../DatasetUpdate/DatasetDedup.spec.ts | 6 +- .../DatasetUpdate/DatasetDenorm.spec.ts | 83 ++----------------- .../DatasetUpdate/Fixtures.ts | 19 +++-- 4 files changed, 22 insertions(+), 91 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 8512af8e..18fe3de4 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -22,6 +22,7 @@ import { druidHttpService } from "../../connections/druidConnection"; import { query, sequelize } from "../../connections/databaseConnection"; import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +export const apiId = "api.datasets.status-transition"; const transitionFailed = "DATASET_STATUS_TRANSITION_FAILURE" const invalidRequest = "DATASET_STATUS_TRANSITION_INVALID_INPUT" const datasetNotFound = "DATASET_NOT_FOUND" @@ -79,10 +80,10 @@ const validateDataset = (req: Request, res: Response, dataset: any, action: stri if(!_.includes(allowedTransitions[action], dataset.status)) { const code = `DATASET_${_.toUpper(action)}_FAILURE` - logger.error({ code, headers, message: `${errorMessage} for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}` }) + logger.error({ code, headers, message: `for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}` }) ResponseHandler.errorResponse({ code: datasetNotFound, - message: `${errorMessage} for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, + message: `$ for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, statusCode: 404, errCode: "NOT_FOUND" } as ErrorObject, req, res); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts index 8d27057e..aa3ae5da 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts @@ -22,7 +22,7 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { it("Success: Dataset dedupe configs updated with dedup key if duplicates need to be dropped", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { it("Success: Dataset dedupe configs updated with default values if duplicates need to be dropped", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -57,7 +57,7 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { chai .request(app) .patch("/v2/datasets/update") - .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, dedup_config: { drop_duplicates: false } } }) + .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, dedup_config: { drop_duplicates: false, dedup_key: "mid" } } }) .end((err, res) => { console.log(res.body.result) res.should.have.status(httpStatus.OK); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index 268b83c8..aaddfd4a 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -22,7 +22,7 @@ describe("DATASET DENORM UPDATE", () => { it("Success: Dataset denorms successfully added", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, denorm_config: { denorm_field: [] } + id: "telemetry", status: "Draft", type:"event", version_key: validVersionKey, api_version:"v2", denorm_config: { denorm_field: [] } }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET DENORM UPDATE", () => { it("Success: Dataset denorms successfully removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, denorm_config: { denorm_fields: [{ denorm_out_field: "userdata" }] } + id: "telemetry", status: "Draft", type:"event", version_key: validVersionKey, api_version:"v2", denorm_config: { denorm_fields: [{ denorm_out_field: "userdata" }] } }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -74,10 +74,11 @@ describe("DATASET DENORM UPDATE", () => { it("Success: When payload contains same denorms to be removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", version_key: validVersionKey, type:"dataset", status: "Draft", denorm_config: { + id: "telemetry", version_key: validVersionKey, type:"dataset", api_version:"v2", status: "Draft", denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }] } }) @@ -102,78 +103,4 @@ describe("DATASET DENORM UPDATE", () => { }); }); - - it("Failure: Dataset contains duplicate denorm field", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ status: "Draft", version_key: validVersionKey }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DUPLICATE_DENORM_KEY) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - expect(res.body.error.message).to.match(/^Dataset contains duplicate denorm out keys(.+)$/) - res.body.error.code.should.be.eq("DATASET_DUPLICATE_DENORM_KEY") - done(); - }); - }); - - it("Failure: When denorm fields provided to add already exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "userdata" - }] - } - }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Denorm fields already exist") - res.body.error.code.should.be.eq("DATASET_DENORM_EXISTS") - done(); - }); - }); - - it("Failure: When denorm fields provided to delete does not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "id" - }] - } - }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Denorm fields do not exist to remove") - res.body.error.code.should.be.eq("DATASET_DENORM_DO_NOT_EXIST") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index 4dd1455e..b5652fa4 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -56,11 +56,12 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "master" }, - "action": "add" + "action": "upsert" } ] } @@ -74,7 +75,7 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, @@ -407,16 +408,18 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }, "action": "remove" }, { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }, "action": "remove" } From ea43a725e68e04e2c2ce2814a84e1009bb8260ea Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 11 Jul 2024 15:35:33 +0530 Subject: [PATCH 019/235] #OBS-I115: Dataset Transition API refactoring and error handling refactoring --- .gitignore | 3 +- api-service/package.json | 2 +- api-service/src/app.ts | 3 +- api-service/src/v2/configs/IngestionConfig.ts | 61 ++++ .../DatasetCreate/DatasetCreate.ts | 39 +-- .../v2/controllers/DatasetList/DatasetList.ts | 16 +- .../v2/controllers/DatasetRead/DatasetRead.ts | 41 +-- .../DatasetStatusTransition.ts | 196 +++---------- .../DatasetUpdate/DatasetUpdate.ts | 75 +---- api-service/src/v2/helpers/ResponseHandler.ts | 11 + api-service/src/v2/middlewares/errors.ts | 8 + api-service/src/v2/services/DatasetService.ts | 103 ++++++- .../src/v2/services/IngestionService.ts | 260 ------------------ api-service/src/v2/services/TableGenerator.ts | 157 +++++++++++ .../DatasetDelete.spec.ts | 5 +- .../DatasetLive.spec.ts | 10 +- .../DatasetReadyToPublish.spec.ts | 9 +- .../DatasetRetire.spec.ts | 19 +- .../DatasetStatusTransition.spec.ts | 6 +- api-service/src/v2/types/ObsrvError.ts | 24 ++ 20 files changed, 457 insertions(+), 591 deletions(-) delete mode 100644 api-service/src/v2/services/IngestionService.ts create mode 100644 api-service/src/v2/services/TableGenerator.ts create mode 100644 api-service/src/v2/types/ObsrvError.ts diff --git a/.gitignore b/.gitignore index 4e1e06fe..5691d838 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ __pycache__/ *.DS_Store #to ignore embedded postgres data files data -connector-registry \ No newline at end of file +connector-registry +package-lock.json diff --git a/api-service/package.json b/api-service/package.json index e4754097..d34a4325 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -70,7 +70,7 @@ "@types/mocha": "^10.0.0", "@types/mock-knex": "^0.4.4", "@types/moment": "^2.13.0", - "@types/node": "^18.11.9", + "@types/node": "^18.19.39", "@types/pg": "^8.6.6", "@types/sinon": "^17.0.3", "@types/slug": "^5.0.8", diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 7783a4aa..2dbbd887 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -10,7 +10,7 @@ import { interceptAuditEvents } from "./v1/services/telemetry"; import { queryService } from "./v1/routes/Router"; import { routesConfig } from "./v1/configs/RoutesConfig"; import { QueryValidator } from "./v1/validators/QueryValidator"; -import { errorHandler } from "./v2/middlewares/errors"; +import { errorHandler, obsrvErrorHandler } from "./v2/middlewares/errors"; const app: Application = express(); const queryValidator = new QueryValidator(); @@ -25,6 +25,7 @@ app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.use(errorHandler) +app.use(obsrvErrorHandler); app.set("queryServices", services); loadExtensions(app) diff --git a/api-service/src/v2/configs/IngestionConfig.ts b/api-service/src/v2/configs/IngestionConfig.ts index 233b9c17..c566be57 100644 --- a/api-service/src/v2/configs/IngestionConfig.ts +++ b/api-service/src/v2/configs/IngestionConfig.ts @@ -1,3 +1,4 @@ +import { config } from "./Config"; const env = process.env; @@ -15,3 +16,63 @@ export const ingestionConfig = { "maxBytesInMemory": env.max_bytes_in_memory || 134217728, "syncts_path": "$.obsrv_meta.syncts", } + +export const rawIngestionSpecDefaults = { + "granularitySpec": { + "type": "uniform", + "rollup": false, + "segmentGranularity": env.segment_granularity || "DAY", + "queryGranularity": "none" + }, + "tuningConfig": { + "type": "kafka", + "maxBytesInMemory": env.max_bytes_in_memory || 134217728, + "maxRowsPerSegment": env.max_rows_per_segment || 5000000, + "logParseExceptions": true + }, + "ioConfig": { + "type": "kafka", + "topic": "", + "consumerProperties": { "bootstrap.servers": config.telemetry_service_config.kafka.config.brokers[0] }, + "taskCount": env.supervisor_task_count || 1, + "replicas": 1, + "taskDuration": env.default_task_duration || "PT4H", + "useEarliestOffset": true, + "completionTimeout": env.default_task_duration || "PT4H", + "inputFormat": { + "type": "json", + "flattenSpec": { + "useFieldDiscovery": true + } + }, + "appendToExisting": false + }, + "synctsField": { + "name": "obsrv_meta.syncts", + "arrival_format": "text", + "data_type": "date", + "expr": "$.obsrv_meta.syncts" + }, + "dimensions": [ + { + "type": "string", + "name": "obsrv.meta.source.connector" + }, + { + "type": "string", + "name": "obsrv.meta.source.id" + } + ], + "flattenSpec": [ + { + "type": "path", + "expr": "$.obsrv_meta.source.['connector']", + "name": "obsrv.meta.source.connector" + }, + { + "type": "path", + "expr": "$.obsrv_meta.source.['connectorInstance']", + "name": "obsrv.meta.source.id" + } + ] +} \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index 17e18943..6b648305 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -1,65 +1,38 @@ import _ from "lodash"; import { Request, Response } from "express"; import httpStatus from "http-status"; -import logger from "../../logger"; import { datasetService } from "../../services/DatasetService"; import DatasetCreate from "./DatasetCreateValidationSchema.json"; import { schemaValidation } from "../../services/ValidationService"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { ErrorObject } from "../../types/ResponseModel"; import { cipherService } from "../../services/CipherService"; import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +import { obsrvError } from "../../types/ObsrvError"; export const apiId = "api.datasets.create" -const isValidRequest = async (req: Request, res: Response): Promise => { +const validateRequest = async (req: Request) => { const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) if (!isRequestValid.isValid) { - logger.error({ code: "DATASET_INVALID_INPUT", apiId, body: req.body, message: isRequestValid.message }) - ResponseHandler.errorResponse({ - code: "DATASET_INVALID_INPUT", - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - return false; + throw obsrvError("", "DATASET_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) } const datasetId = _.get(req, ["body", "request", "dataset_id"]) const isDataSetExists = await datasetService.checkDatasetExists(datasetId); if (isDataSetExists) { - logger.error({ code: "DATASET_EXISTS", apiId, body: req.body, message: `Dataset Already exists with id:${datasetId}` }) - ResponseHandler.errorResponse({ - code: "DATASET_EXISTS", - message: "Dataset already exists", - statusCode: 409, - errCode: "CONFLICT" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, "DATASET_EXISTS", `Dataset Already exists with id:${datasetId}`, "CONFLICT", 409) } const duplicateDenormKeys = datasetService.getDuplicateDenormKey(_.get(req, ["body", "request", "denorm_config"])) if (!_.isEmpty(duplicateDenormKeys)) { - const code = "DATASET_DUPLICATE_DENORM_KEY" - logger.error({ code: "DATASET_DUPLICATE_DENORM_KEY", apiId, body: req.body, message: `Duplicate denorm output fields found. Duplicate Denorm out fields are [${duplicateDenormKeys}]` }) - ResponseHandler.errorResponse({ - code: "DATASET_DUPLICATE_DENORM_KEY", - statusCode: 400, - message: "Duplicate denorm key found", - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, "DATASET_DUPLICATE_DENORM_KEY", "Duplicate denorm output fields found.", "BAD_REQUEST", 400, undefined, {duplicateKeys: duplicateDenormKeys}) } - return true; } const datasetCreate = async (req: Request, res: Response) => { - const isRequestValid = await isValidRequest(req, res) - if(!isRequestValid) { - return; - } + validateRequest(req) const draftDataset = getDraftDataset(req.body.request) const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); diff --git a/api-service/src/v2/controllers/DatasetList/DatasetList.ts b/api-service/src/v2/controllers/DatasetList/DatasetList.ts index f73854ca..81b0056e 100644 --- a/api-service/src/v2/controllers/DatasetList/DatasetList.ts +++ b/api-service/src/v2/controllers/DatasetList/DatasetList.ts @@ -5,8 +5,8 @@ import logger from "../../logger"; import { schemaValidation } from "../../services/ValidationService"; import DatasetCreate from "./DatasetListValidationSchema.json"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { ErrorObject } from "../../types/ResponseModel"; import { datasetService } from "../../services/DatasetService"; +import { obsrvError } from "../../types/ObsrvError"; export const apiId = "api.datasets.list" export const errorCode = "DATASET_LIST_FAILURE" @@ -16,25 +16,15 @@ const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version" const datasetList = async (req: Request, res: Response) => { - const requestBody = req.body; - const msgid = _.get(req, ["body", "params", "msgid"]); - const resmsgid = _.get(res, "resmsgid"); const isRequestValid: Record = schemaValidation(req.body, DatasetCreate) if (!isRequestValid.isValid) { - const code = "DATASET_LIST_INPUT_INVALID" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); + throw obsrvError("", "DATASET_LIST_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) } const datasetBody = req.body.request; const datasetList = await listDatasets(datasetBody) const responseData = { data: datasetList, count: _.size(datasetList) } - logger.info({ apiId, msgid, requestBody, resmsgid, message: `Datasets are listed successfully with a dataset count (${_.size(datasetList)})` }) + logger.info({req: req.body, resmsgid: _.get(res, "resmsgid"), message: `Datasets are listed successfully with a dataset count (${_.size(datasetList)})` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); } diff --git a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts index 90e14537..c82008fc 100644 --- a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts @@ -1,11 +1,10 @@ import { Request, Response } from "express"; import httpStatus from "http-status"; import _ from "lodash"; -import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { ErrorObject } from "../../types/ResponseModel"; import { DatasetDraft } from "../../models/DatasetDraft"; import { datasetService } from "../../services/DatasetService"; +import { obsrvError } from "../../types/ObsrvError"; export const apiId = "api.datasets.read"; export const errorCode = "DATASET_READ_FAILURE" @@ -13,52 +12,30 @@ export const errorCode = "DATASET_READ_FAILURE" // TODO: Move this to a config const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config"] -const isValidRequest = (req: Request, res: Response): boolean => { +const validateRequest = (req: Request) => { + const { dataset_id } = req.params; const fields = req.query.fields; if(fields && typeof fields !== 'string') { - logger.error({ code: "DATASET_INVALID_FIELDS_VAL", apiId, dataset_id, message: `The specified fields [${fields}] in the query param is not a string.` }) - ResponseHandler.errorResponse({ - code: "DATASET_INVALID_FIELDS_VAL", - message: `The specified fields [${fields}] in the query param is not a string.`, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - return false; + throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS_VAL", `The specified fields [${fields}] in the query param is not a string.`, "BAD_REQUEST", 400); } - const fieldValues = fields ? _.split(fields, ",") : [] - const invalidFields = _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes())) + const fieldValues = fields ? _.split(fields, ",") : []; + const invalidFields = _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes())); if (!_.isEmpty(invalidFields)) { - logger.error({ code: "DATASET_INVALID_FIELDS", apiId, dataset_id, message: `The specified fields [${invalidFields}] in the dataset cannot be found` }) - ResponseHandler.errorResponse({ - code: "DATASET_INVALID_FIELDS", - message: `The specified fields [${invalidFields}] in the dataset cannot be found.`, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - return false; + throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS", `The specified fields [${invalidFields}] in the dataset cannot be found.`, "BAD_REQUEST", 400); } - return true; } const datasetRead = async (req: Request, res: Response) => { - if(!isValidRequest(req, res)) { - return; - } + validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; const attributes = !fields ? defaultFields : _.split(fields, ","); const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes) if(!dataset) { - logger.error({ code: "DATASET_NOT_FOUND", apiId, dataset_id, message: `Dataset with the given dataset_id:${dataset_id} not found` }) - ResponseHandler.errorResponse({ - code: "DATASET_NOT_FOUND", - message: "Dataset with the given dataset_id not found", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); + throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } else { ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 8512af8e..a9592233 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -1,28 +1,16 @@ import { Request, Response } from "express"; import _ from "lodash"; -import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { datasetService } from "../../services/DatasetService"; -import { ErrorObject } from "../../types/ResponseModel"; import { schemaValidation } from "../../services/ValidationService"; import StatusTransitionSchema from "./RequestValidationSchema.json"; import ReadyToPublishSchema from "./ReadyToPublishSchema.json" import httpStatus from "http-status"; -import { DatasetTransformationsDraft } from "../../models/TransformationDraft"; -import { DatasourceDraft } from "../../models/DatasourceDraft"; -import { DatasetSourceConfigDraft } from "../../models/DatasetSourceConfigDraft"; -import { DatasetDraft } from "../../models/DatasetDraft"; -import { Dataset } from "../../models/Dataset"; -import { DatasetAction, DatasetStatus, DatasetType } from "../../types/DatasetModels"; -import { DatasetSourceConfig } from "../../models/DatasetSourceConfig"; -import { Datasource } from "../../models/Datasource"; -import { DatasetTransformations } from "../../models/Transformation"; +import { DatasetStatus, DatasetType } from "../../types/DatasetModels"; import { executeCommand } from "../../connections/commandServiceConnection"; -import { druidHttpService } from "../../connections/druidConnection"; -import { query, sequelize } from "../../connections/databaseConnection"; import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +import { obsrvError } from "../../types/ObsrvError"; -const transitionFailed = "DATASET_STATUS_TRANSITION_FAILURE" const invalidRequest = "DATASET_STATUS_TRANSITION_INVALID_INPUT" const datasetNotFound = "DATASET_NOT_FOUND" @@ -36,57 +24,25 @@ const allowedTransitions: Record = { } const liveDatasetActions = ["Retire", "Archive", "Purge"] -const statusTransitionCommands = { - Delete: ["DELETE_DRAFT_DATASETS"], - ReadyToPublish: ["VALIDATE_DATASET_CONFIGS"], - Live: ["PUBLISH_DATASET"], - Retire: ["CHECK_DATASET_IS_DENORM", "SET_DATASET_TO_RETIRE", "DELETE_SUPERVISORS", "RESTART_PIPELINE"] -} - -const logHeaders = (req: Request, res: Response) => { - return { - apiId: "api.datasets.status-transition", msgid:_.get(req, ["body", "params", "msgid"]), request: req.body, resmsgid: _.get(res, "resmsgid") - } -} - -const validateRequest = (req: Request, res: Response, headers: Record): boolean => { +const validateRequest = (req: Request, datasetId: any) => { const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) if (!isRequestValid.isValid) { - logger.error({ code: invalidRequest, headers, message: isRequestValid.message }) - ResponseHandler.errorResponse({ - code: invalidRequest, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, invalidRequest, isRequestValid.message, "BAD_REQUEST", 400) } - return true; } -const validateDataset = (req: Request, res: Response, dataset: any, action: string, headers: Record) : boolean => { +const validateDataset = (dataset: any, datasetId: any, action: string) => { if (_.isEmpty(dataset)) { - logger.error({ code: datasetNotFound, headers, message: `Dataset not found for dataset:${dataset.id}` }) - ResponseHandler.errorResponse({ - code: datasetNotFound, - message: `Dataset not found for dataset: ${dataset.id}`, - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, datasetNotFound, `Dataset not found for dataset: ${dataset.id}`, "NOT_FOUND", 404) + } + + if (dataset.api_version !== "v2" && _.includes(["ReadyToPublish", "Live"], action)) { + throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "NOT_FOUND", 404) } if(!_.includes(allowedTransitions[action], dataset.status)) { - const code = `DATASET_${_.toUpper(action)}_FAILURE` - logger.error({ code, headers, message: `${errorMessage} for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}` }) - ResponseHandler.errorResponse({ - code: datasetNotFound, - message: `${errorMessage} for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, `DATASET_${_.toUpper(action)}_FAILURE`, `Transition failed for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, "NOT_FOUND", 404) } return true; @@ -95,17 +51,11 @@ const validateDataset = (req: Request, res: Response, dataset: any, action: stri const datasetStatusTransition = async (req: Request, res: Response) => { - const headers = logHeaders(req, res) const { dataset_id, status } = _.get(req.body, "request"); - if (!validateRequest(req, res, headers)) { - return; - } + validateRequest(req, dataset_id); const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type"]) - - if(!validateDataset(req, res, dataset, status, headers)) { - return; - } + validateDataset(dataset, dataset_id, status); switch(status) { case "Delete": @@ -128,28 +78,15 @@ const datasetStatusTransition = async (req: Request, res: Response) => { break; } - logger.info({ headers, message: `Dataset status transition to ${status} successful with id:${dataset_id}` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: `Dataset status transition to ${status} successful`, dataset_id } }); - } // Delete a draft dataset const deleteDataset = async (dataset: Record) => { + await datasetService.deleteDraftDataset(dataset) // TODO: Delete any sample files or schemas that are uploaded - const { id } = dataset - const transaction = await sequelize.transaction() - try { - await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetDraft.destroy({ where: { id } , transaction}) - await transaction.commit() - } catch (err) { - await transaction.rollback() - throw err - } } @@ -166,25 +103,28 @@ const readyForPublish = async (dataset: Record) => { } } _.set(draftDataset, 'status', DatasetStatus.ReadyToPublish) - await DatasetDraft.update(draftDataset, { where: { id: dataset.id } }) -} - - - -//PUBLISH_DATASET + await datasetService.updateDraftDataset(draftDataset) +} + +/** + * Method to publish a dataset. Does the following: + * 1. Validate if all the denorm datasets are valid, no cirular reference and are in Live status + * 2. Update the redis host and db if the dataset is a master dataset + * 3. Save the draft copy + * 4. Generate the Druid and Hudi datasource configuration depending on the storage configured + * + * @param dataset + */ const publishDataset = async (dataset: Record) => { - const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record await validateAndUpdateDenormConfig(draftDataset); - await updateMaterDataConfig(draftDataset) - await DatasetDraft.update(draftDataset, { where: { id: dataset.id } }) - await generateDataSouce(draftDataset) - - await executeCommand(dataset.id, "PUBLISH_DATASET"); + await updateMasterDataConfig(draftDataset) + await datasetService.publishDataset(draftDataset) } -const validateAndUpdateDenormConfig = async (draftDataset: any) => { +const validateAndUpdateDenormConfig = async (draftDataset: Record) => { // 1. Check if there are denorm fields and dependent master datasets are published const denormConfig = _.get(draftDataset, "denorm_config") @@ -234,10 +174,10 @@ const validateAndUpdateDenormConfig = async (draftDataset: any) => { } } -const updateMaterDataConfig = async (draftDataset: any) => { +const updateMasterDataConfig = async (draftDataset: Record) => { if(draftDataset.type === 'master') { if(draftDataset.dataset_config.cache_config.redis_db === 0) { - const { results }: any = await query("SELECT nextval('redis_db_index')") + const { results }: any = await datasetService.getNextRedisDBIndex() if(_.isEmpty(results)) { throw { code: "REDIS_DB_INDEX_FETCH_FAILED", @@ -252,15 +192,10 @@ const updateMaterDataConfig = async (draftDataset: any) => { } } -const generateDataSouce = async (draftDataset: any) => { - -} - const retireDataset = async (dataset: Record) => { await canRetireIfMasterDataset(dataset); - await updateDatasetStatusToRetired(dataset); - await deleteDruidSupervisors(dataset); + await datasetService.retireDataset(dataset); await restartPipeline(dataset); } @@ -269,86 +204,33 @@ const canRetireIfMasterDataset = async (dataset: Record) => { if (dataset.type === DatasetType.master) { - const liveDatasets = await Dataset.findAll({ where: { status: DatasetStatus.Live}, attributes: ["denorm_config", "id", "status"], raw: true }) || [] - const draftDatasets = await DatasetDraft.findAll({ where: { status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, attributes: ["denorm_config", "id", "status"], raw: true }) || [] + const liveDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live }, ["denorm_config", "id", "status"]) || [] + const draftDatasets = await datasetService.findDraftDatasets({ status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, ["denorm_config", "id", "status"]) || [] const allDatasets = _.union(liveDatasets, draftDatasets) const extractDenormFields = _.map(allDatasets, function(depDataset) { return {dataset_id: _.get(depDataset, 'id'), status: _.get(depDataset, 'status'), denorm_datasets: _.map(_.get(depDataset, 'denorm_config.denorm_fields'), 'dataset_id')} }) const deps = _.filter(extractDenormFields, function(depDS) { return _.includes(depDS.denorm_datasets, dataset.id)}) if (_.size(deps) > 0) { - const denormErrMsg = `Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset` - logger.error(denormErrMsg); - throw { - code: "DATASET_IN_USE", - errCode: "BAD_REQUEST", - message: denormErrMsg, - statusCode: 400, - data: _.map(deps, function(o) { return _.omit(o, 'denorm_datasets')}) - } - } - } -} -const updateDatasetStatusToRetired = async (dataset: Record) => { - - const transaction = await sequelize.transaction() - try { - await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }) - await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }) - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}) - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}) - await transaction.commit() - } catch(err:any) { - await transaction.rollback() - throw { - code: "UNABLE_TO_RETIRE_DATASET", - errCode: "SERVER_ERROR", - message: err.message, - statusCode: 500 - } - } -} - -const deleteDruidSupervisors = async (dataset: Record) => { - - try { - if (dataset.type !== DatasetType.master) { - const datasourceRefs = await Datasource.findAll({ where: { dataset_id: dataset.id }, attributes: ["datasource_ref"], raw: true }) - for (const sourceRefs of datasourceRefs) { - const datasourceRef = _.get(sourceRefs, "datasource_ref") - await druidHttpService.post(`/druid/indexer/v1/supervisor/${datasourceRef}/terminate`) - logger.info(`Datasource ref ${datasourceRef} deleted from druid`) - } + const denormErrMsg = `Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset` + throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function(o) { return _.omit(o, 'denorm_datasets')})) } - } catch (error: any) { - logger.error({ error: _.get(error, "message"), message: `Failed to delete supervisors for dataset:${dataset.id}` }) } } -//RESTART_PIPELINE const restartPipeline = async (dataset: Record) => { return executeCommand(dataset.id, "RESTART_PIPELINE") } const archiveDataset = async (dataset: Record) => { - throw { - code: "ARCHIVE_NOT_IMPLEMENTED", - errCode: "NOT_IMPLEMENTED", - message: "Archive functionality is not implemented", - statusCode: 501 - } + throw obsrvError(dataset.id, "ARCHIVE_NOT_IMPLEMENTED", "Archive functionality is not implemented", "NOT_IMPLEMENTED", 501) } const purgeDataset = async (dataset: Record) => { - throw { - code: "PURGE_NOT_IMPLEMENTED", - errCode: "NOT_IMPLEMENTED", - message: "Purge functionality is not implemented", - statusCode: 501 - } + throw obsrvError(dataset.id, "PURGE_NOT_IMPLEMENTED", "Purge functionality is not implemented", "NOT_IMPLEMENTED", 501) } export default datasetStatusTransition; \ No newline at end of file diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts index f5e68db4..1a8d68a5 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts @@ -1,111 +1,60 @@ import { Request, Response } from "express"; import httpStatus from "http-status"; -import logger from "../../logger"; import _ from "lodash"; import Model from "sequelize/types/model"; -import { ErrorObject } from "../../types/ResponseModel"; import { DatasetStatus } from "../../types/DatasetModels"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { cipherService } from "../../services/CipherService"; import { datasetService } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import DatasetUpdate from "./DatasetUpdateValidationSchema.json"; +import { obsrvError } from "../../types/ObsrvError"; export const apiId = "api.datasets.update"; export const invalidInputErrCode = "DATASET_UPDATE_INPUT_INVALID" export const errorCode = "DATASET_UPDATE_FAILURE" -const isValidRequest = async (req: Request, res: Response): Promise => { +const validateRequest = async (req: Request) => { + const datasetId = _.get(req, ["body", "request", "dataset_id"]) const isRequestValid: Record = schemaValidation(req.body, DatasetUpdate) if (!isRequestValid.isValid) { - logger.error({ code: "DATASET_UPDATE_INPUT_INVALID", apiId, body: req.body, message: isRequestValid.message }) - ResponseHandler.errorResponse({ - code: "DATASET_UPDATE_INPUT_INVALID", - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, "DATASET_UPDATE_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) } const datasetBody = req.body.request const { dataset_id, version_key, ...rest } = datasetBody if (_.isEmpty(rest)) { - const code = "DATASET_UPDATE_NO_FIELDS" - logger.error({ code, apiId, body: req.body, message: `Provide atleast one field in addition to the dataset_id:${dataset_id} and version_key:${version_key} to update the dataset` }) - ResponseHandler.errorResponse({ - code, - message: "Provide atleast one field in addition to the dataset_id to update the dataset", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, "DATASET_UPDATE_NO_FIELDS", "Provide atleast one field in addition to the dataset_id to update the dataset", "BAD_REQUEST", 400) } - return true; } -const isValidDataset = (dataset: Record | null, req: Request, res: Response): boolean => { +const validateDataset = (dataset: Record | null, req: Request) => { const datasetId = _.get(req, ["body", "request", "dataset_id"]) const versionKey = _.get(req, ["body", "request", "version_key"]) if (dataset) { if (dataset.api_version !== "v2") { - logger.error({ code: "DATASET_API_VERSION_MISMATCH", apiId, body: req.body, message: `Draft dataset api version is not v2:${datasetId}` }) - ResponseHandler.errorResponse({ - code: "DATASET_API_VERSION_MISMATCH", - message: "Draft dataset api version is not v2", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "NOT_FOUND", 404) } if(dataset.version_key !== versionKey) { - logger.error({ code: "DATASET_OUTDATED", body: req.body, message: `The dataset:${datasetId} with version_key:${versionKey} is outdated. Please try to fetch latest changes of the dataset with version key:${dataset.version_key} and perform the updates` }) - ResponseHandler.errorResponse({ - code: "DATASET_OUTDATED", - message: "The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates", - statusCode: 409, - errCode: "CONFLICT" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, "DATASET_OUTDATED", "The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates", "CONFLICT", 409) } if(!_.includes([DatasetStatus.Draft, DatasetStatus.ReadyToPublish], dataset.status)) { - logger.error({ code: "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE", body: req.body, message: `Dataset with id:${datasetId} cannot be updated as it is not in draft state` }) - ResponseHandler.errorResponse({ - code: "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE", - message: "Dataset cannot be updated as it is not in draft state", - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, "DATASET_NOT_IN_DRAFT_STATE_TO_UPDATE", "Dataset cannot be updated as it is not in draft state", "BAD_REQUEST", 400) } - return true; } else { - logger.error({ code: "DATASET_NOT_EXISTS", body: req.body, message: `Dataset does not exists with id:${datasetId}` }) - ResponseHandler.errorResponse({ - code: "DATASET_NOT_EXISTS", - message: "Dataset does not exists to update", - statusCode: 404, - errCode: "NOT_FOUND" - } as ErrorObject, req, res); - return false; + throw obsrvError(datasetId, "DATASET_NOT_EXISTS", `Dataset does not exists with id:${datasetId}`, "NOT_FOUND", 404) } } const datasetUpdate = async (req: Request, res: Response) => { - const isRequestValid = await isValidRequest(req, res) - if(!isRequestValid) { - return; - } - + await validateRequest(req) const datasetReq = req.body.request; const datasetModel = await datasetService.getDraftDataset(datasetReq.dataset_id) - if(!isValidDataset(datasetModel, req, res)) { - return; - } + validateDataset(datasetModel, req) const draftDataset = mergeDraftDataset(datasetModel, datasetReq); const response = await datasetService.updateDraftDataset(draftDataset); diff --git a/api-service/src/v2/helpers/ResponseHandler.ts b/api-service/src/v2/helpers/ResponseHandler.ts index c8d7e8b1..faeb3847 100644 --- a/api-service/src/v2/helpers/ResponseHandler.ts +++ b/api-service/src/v2/helpers/ResponseHandler.ts @@ -4,6 +4,7 @@ import { IResponse, Result } from "../types/DatasetModels"; import { onFailure, onSuccess } from "../metrics/prometheus/helpers"; import moment from "moment"; import _ from "lodash"; +import { ObsrvError } from "../types/ObsrvError"; const ResponseHandler = { successResponse: (req: Request, res: Response, result: Result) => { @@ -34,6 +35,16 @@ const ResponseHandler = { entity && onFailure(req, res) }, + obsrvErrorResponse: (error: ObsrvError, req: Request, res: Response) => { + const { statusCode, message, errCode, code = "INTERNAL_SERVER_ERROR", data } = error; + const { id, entity, body } = req as any; + const msgid = _.get(body, ["params", "msgid"]) + const resmsgid = _.get(res, "resmsgid") + const response = ResponseHandler.refactorResponse({ id, msgid, params: { status: "FAILED" }, responseCode: errCode || httpStatus["500_NAME"], resmsgid, result: data }) + res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json({ ...response, error: { code, message } }); + entity && onFailure(req, res) + }, + setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { (req as any).id = id; next(); diff --git a/api-service/src/v2/middlewares/errors.ts b/api-service/src/v2/middlewares/errors.ts index e52f05f3..568aaa67 100644 --- a/api-service/src/v2/middlewares/errors.ts +++ b/api-service/src/v2/middlewares/errors.ts @@ -2,10 +2,18 @@ import { NextFunction, Request, Response } from "express"; import logger from "../logger"; import { ResponseHandler } from "../helpers/ResponseHandler"; import _ from "lodash"; +import { ObsrvError } from "../types/ObsrvError"; export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => { logger.error({ path: req.url, req: req.body , ...err }) let errorMessage = {name: err.name, message: err.message}; ResponseHandler.errorResponse(errorMessage, req, res); +}; + + +export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, next: NextFunction) => { + + logger.error({ path: req.url, req: req.body, resmsgid: _.get(res, "resmsgid") , ...obsrvErr }) + ResponseHandler.obsrvErrorResponse(obsrvErr, req, res); }; \ No newline at end of file diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 68d1da27..18a608a8 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -6,9 +6,17 @@ import { DatasetTransformations } from "../models/Transformation"; import { DatasetTransformationsDraft } from "../models/TransformationDraft"; import Model from "sequelize/types/model"; import { DatasetSourceConfigDraft } from "../models/DatasetSourceConfigDraft"; -import { sequelize } from "../connections/databaseConnection"; +import { query, sequelize } from "../connections/databaseConnection"; import { DatasetSourceConfig } from "../models/DatasetSourceConfig"; import { ConnectorInstances } from "../models/ConnectorInstances"; +import { DatasourceDraft } from "../models/DatasourceDraft"; +import { executeCommand } from "../connections/commandServiceConnection"; +import Transaction from "sequelize/types/transaction"; +import { DatasetStatus, DatasetType } from "../types/DatasetModels"; +import { Datasource } from "../models/Datasource"; +import { obsrvError } from "../types/ObsrvError"; +import { druidHttpService } from "../connections/druidConnection"; +import { tableGenerator } from "./TableGenerator"; class DatasetService { @@ -181,6 +189,99 @@ class DatasetService { return await this.getDraftDataset(draftDataset.dataset_id); } + getNextRedisDBIndex = async () => { + return await query("SELECT nextval('redis_db_index')") + } + + deleteDraftDataset = async (dataset: Record) => { + + const { id } = dataset + const transaction = await sequelize.transaction() + try { + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) + await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) + await DatasetDraft.destroy({ where: { id } , transaction}) + await transaction.commit() + } catch (err:any) { + await transaction.rollback() + throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) + } + } + + retireDataset = async (dataset: Record) => { + + const transaction = await sequelize.transaction(); + try { + await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); + await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); + await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); + await transaction.commit(); + await this.deleteDruidSupervisors(dataset); + } catch(err:any) { + await transaction.rollback(); + throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); + } + } + + deleteDruidSupervisors = async (dataset: Record) => { + + try { + if (dataset.type !== DatasetType.master) { + const datasourceRefs = await Datasource.findAll({ where: { dataset_id: dataset.id }, attributes: ["datasource_ref"], raw: true }) + for (const sourceRefs of datasourceRefs) { + const datasourceRef = _.get(sourceRefs, "datasource_ref") + await druidHttpService.post(`/druid/indexer/v1/supervisor/${datasourceRef}/terminate`) + logger.info(`Datasource ref ${datasourceRef} deleted from druid`) + } + } + } catch (error: any) { + logger.error({ error: _.get(error, "message"), message: `Failed to delete supervisors for dataset:${dataset.id}` }) + } + } + + publishDataset = async (draftDataset: Record) => { + + const transaction = await sequelize.transaction() + try { + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } , transaction}) + await this.generateDataSouce(draftDataset, transaction) + await transaction.commit() + await executeCommand(draftDataset.id, "PUBLISH_DATASET"); + } catch(err:any) { + await transaction.rollback() + throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); + } + + } + + generateDataSouce = async (draftDataset: Record, transaction: Transaction) => { + + const indexingConfig = draftDataset.dataset_config.indexing_config; + if(indexingConfig.olap_store_enabled) { + const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); + const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, draftDatasource.datasource_ref); + _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + await DatasourceDraft.create(draftDatasource, {transaction}) + } + if(indexingConfig.lakehouse_enabled) { + + } + } + + createDraftDatasource = (draftDataset: Record, type: string) : Record => { + + const datasource = _.join([draftDataset.dataset_id,"events"], "_") + return { + id: datasource, + datasource: draftDataset.dataset_id, + dataset_id: draftDataset.dataset_id, + datasource_ref: datasource, + type + } + } + } export const datasetService = new DatasetService(); \ No newline at end of file diff --git a/api-service/src/v2/services/IngestionService.ts b/api-service/src/v2/services/IngestionService.ts deleted file mode 100644 index 9ab64de6..00000000 --- a/api-service/src/v2/services/IngestionService.ts +++ /dev/null @@ -1,260 +0,0 @@ -import _ from "lodash"; -import { ingestionConfig } from "../configs/IngestionConfig"; -import { config } from "../configs/Config"; -import { ErrorObject } from "../types/ResponseModel"; -import logger from "../logger"; -import { IngestionSpecModel, IngestionSpecObject } from "../types/IngestionModels"; -const defaultIndexCol = ingestionConfig.indexCol["Event Arrival Time"] - -const connectorSpecObj = { - "flattenSpec": { - "type": "path", - "expr": "$.obsrv_meta.source.['connector']", - "name": "obsrv.meta.source.connector" - }, - "dimensions": { - "type": "string", - "name": "obsrv.meta.source.connector" - }, - "fieldType": "dimensions" -} - -const connectorInstanceSpecObj = { - "flattenSpec": { - "type": "path", - "expr": "$.obsrv_meta.source.['connectorInstance']", - "name": "obsrv.meta.source.id" - }, - "dimensions": { - "type": "string", - "name": "obsrv.meta.source.id" - }, - "fieldType": "dimensions" -} - -export const generateIngestionSpec = (payload: Record) => { - const { indexCol = defaultIndexCol, data_schema, id, dataset_id } = payload - const isValidTimestamp = checkTimestampCol({ indexCol, data_schema }) - if (!isValidTimestamp) { - throw { - "code": "DATASET_TIMESTAMP_NOT_FOUND", - "message": "Provided timestamp key not found in the data schema", - "statusCode": 400, - "errCode": "BAD_REQUEST" - } as ErrorObject - } - const simplifiedSpec = generateExpression(_.get(data_schema, "properties"), indexCol); - const generatedSpec = process(simplifiedSpec, indexCol) - const ingestionTemplate = generateIngestionTemplate({ generatedSpec, id, indexCol, dataset_id, type: "druid" }) - return ingestionTemplate -} - -const generateIngestionTemplate = (payload: Record) => { - const { type, ...rest } = payload - switch (type) { - case "druid": - return getDruidIngestionTemplate(rest); - default: - return null; - } -} - -const checkTimestampCol = (schema: Record) => { - const { indexCol, data_schema } = schema - if (indexCol !== defaultIndexCol) { - const properties = generateFlattenSchema(data_schema); - const exists = _.find(properties, (value) => `$.${indexCol}` === value.path); - return exists - } - return true -} - -const process = (spec: Map, indexCol: string): IngestionSpecModel => { - const colValues = Array.from(spec.values()) - const dimensions = filterDimensionCols(colValues) - return { - "dimensions": getObjByKey(dimensions, "dimensions"), - "metrics": filterMetricsCols(spec), - "flattenSpec": filterFlattenSpec(colValues, indexCol) - } -} - -const filterFlattenSpec = (column: Record, indexCol: string) => { - let flattenSpec = getObjByKey(column, "flattenSpec") - if (indexCol === defaultIndexCol) { - const indexColDefaultSpec = { - "expr": ingestionConfig.syncts_path, - "name": ingestionConfig.indexCol["Event Arrival Time"], - "type": "path" - } - flattenSpec = _.concat(flattenSpec, indexColDefaultSpec) - } - return flattenSpec -} - -const filterMetricsCols = (spec: Record) => { - const metrics = _.filter(spec, cols => cols.fieldType === "metrics") - const metricCols = _.map(metrics, (value) => value["dimensions"]) - const updatedMetrics: any[] = [] - _.map(metricCols, (value) => { - value["fieldName"] = value["fieldName"] || value["name"]; - updatedMetrics.push(value); - }); - return updatedMetrics; -} - -const filterDimensionCols = (spec: Record) => { - const dimensionCols = _.filter(spec, cols => cols.fieldType === "dimensions") - return dimensionCols -} - -const getObjByKey = (sample: any, key: string) => { - const result = _.map(sample, (value) => _.get(value, key)); - return _.compact(result) -} - -export const generateFlattenSchema = (sample: Map) => { - const array: any[] = []; - const flattenValues = (data: any, path: string) => { - _.map(data, (value, key) => { - if (_.isPlainObject(value) && _.has(value, "properties")) { - array.push(flattenSchema(key, `${path}.${key}`)) - flattenValues(value["properties"], `${path}.${key}`); - } else if (_.isPlainObject(value)) { - if (value.type === "array") { - array.push(flattenSchema(key, `${path}.${key}`)) - if (_.has(value, "items") && _.has(value["items"], "properties")) { - flattenValues(value["items"]["properties"], `${path}.${key}`); - } - } else { - array.push(flattenSchema(key, `${path}.${key}`)) - } - } - }) - } - const properties = _.get(sample, "properties") - flattenValues(properties, "$") - return array -} - -const flattenSchema = (expr: string, path: string) => { - return { "property": expr, "path": path } -} - -export const generateExpression = (sample: Map, indexCol: string): Map => { - const flattendedSchema = new Map(); - const flattenExpression = (data: any, path: string) => { - _.map(data, (value, key) => { - if (_.isPlainObject(value) && (_.has(value, "properties"))) { - flattenExpression(value["properties"], `${path}.${key}`); - } else if (_.isPlainObject(value)) { - if (value.type === "array") { - if (_.has(value, "items") && _.has(value["items"], "properties")) { - flattenExpression(value["items"]["properties"], `${path}.${key}[*]`); - } else { - const objectType = getObjectType(value.type) - const specObject = createSpecObj({ expression: `${path}.['${key}'][*]`, objectType, name: `${path}.${key}`, indexCol }) - flattendedSchema.set(`${path}.${key}`, specObject) - } - } else if (value.type == "object" && (!_.has(value, "properties"))) { - const objectType = getObjectType(value.type) - const specObject = createSpecObj({ expression: `${path}.['${key}']`, objectType, name: `${path}.${key}`, indexCol }) - flattendedSchema.set(`${path}.${key}`, specObject) - logger.warn(`Found empty object without properties in the schema with Key: ${key}, Object: ${JSON.stringify(value)}`) - } - else { - const objectType = getObjectType(value.type) - const specObject = createSpecObj({ expression: `${path}.['${key}']`, objectType, name: `${path}.${key}`, indexCol }) - flattendedSchema.set(`${path}.${key}`, specObject) - } - } - }) - } - flattenExpression(sample, "$") - flattendedSchema.set("obsrv.meta.source.connector", connectorSpecObj).set("obsrv.meta.source.connector.instance", connectorInstanceSpecObj) - return flattendedSchema -} - -const createSpecObj = (payload: Record): IngestionSpecObject => { - const { expression, objectType, name, indexCol } = payload - const propertyName = _.replace(name.replace("[*]", ""), "$.", "") - const specObj = { - "flattenSpec": { - "type": "path", - "expr": expression, - "name": propertyName - }, - "dimensions": { - "type": objectType, - "name": propertyName - }, - "fieldType": "dimensions" - } - if ([indexCol].includes(specObj.flattenSpec.name)) { - specObj.fieldType = "timestamp" - } - return specObj -} - -const getObjectType = (type: string): string => { - switch (type) { - case "number": return "double"; - case "integer": return "long"; - case "object": return "json"; - case "boolean": return "string"; - default: return type; - } -} - -export const getDruidIngestionTemplate = (payload: Record) => { - const { id, generatedSpec, indexCol, dataset_id } = payload - const { dimensions, metrics, flattenSpec } = generatedSpec - const dataSource = `${id}_${_.toLower(ingestionConfig.granularitySpec.segmentGranularity)}` - return { - "type": "kafka", - "spec": { - "dataSchema": { - "dataSource": dataSource, - "dimensionsSpec": { "dimensions": dimensions }, - "timestampSpec": { "column": indexCol, "format": "auto" }, - "metricsSpec": metrics, - "granularitySpec": getGranularityObj(), - }, - "tuningConfig": { - "type": "kafka", - "maxBytesInMemory": ingestionConfig.maxBytesInMemory, - "maxRowsPerSegment": ingestionConfig.tuningConfig.maxRowPerSegment, - "logParseExceptions": true - }, - "ioConfig": getIOConfigObj(flattenSpec, dataset_id) - } - } -} - -const getGranularityObj = () => { - return { - "type": "uniform", - "segmentGranularity": ingestionConfig.granularitySpec.segmentGranularity, - "queryGranularity": ingestionConfig.query_granularity, - "rollup": ingestionConfig.granularitySpec.rollup - } -} - -const getIOConfigObj = (flattenSpec: Record, topic: string): Record => { - return { - "type": "kafka", - "topic": topic, - "consumerProperties": { "bootstrap.servers": config.telemetry_service_config.kafka.config.brokers[0] }, - "taskCount": ingestionConfig.tuningConfig.taskCount, - "replicas": 1, - "taskDuration": ingestionConfig.ioconfig.taskDuration, - "useEarliestOffset": ingestionConfig.use_earliest_offset, - "completionTimeout": ingestionConfig.completion_timeout, - "inputFormat": { - "type": "json", "flattenSpec": { - "useFieldDiscovery": true, "fields": flattenSpec - } - }, - "appendToExisting": false - } -} \ No newline at end of file diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts new file mode 100644 index 00000000..7772b1b3 --- /dev/null +++ b/api-service/src/v2/services/TableGenerator.ts @@ -0,0 +1,157 @@ +import _ from "lodash"; +import { rawIngestionSpecDefaults } from "../configs/IngestionConfig"; +import { datasetService } from "./DatasetService"; + +class BaseTableGenerator { + + /** + * Method to flatten a json schema - extract all properties using jsonpath notation + * + * @param dataSchema + * @returns properties Record[] + */ + flattenSchema = (dataSchema: Record) : Record[] => { + + let properties: Record[] = [] + const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { + _.mapKeys(schema, function(value, parentKey) { + const newKey = (prev) ? _.join([prev, parentKey], '.') : parentKey; + const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], '') : _.join(["$.['", parentKey, "']"], ''); + switch(value['type']) { + case 'object': + flatten(_.get(value, 'properties'), newKey, newExpr); + break; + case 'array': + if(_.get(value, 'items.type') == 'object' && _.get(value, 'items.properties')) { + _.mapKeys(_.get(value, 'items.properties'), function(value, childKey) { + const objChildKey = _.join([newKey, childKey], '.') + properties.push(_.merge(_.pick(value, ['arrival_format', 'is_deleted']), {expr: _.join([newExpr,"[*].['",childKey,"']"], ''), name: objChildKey, data_type: 'array'})) + }) + } else { + properties.push(_.merge(_.pick(value, ['arrival_format', 'data_type', 'is_deleted']), {expr: newExpr, name: newKey})) + } + break; + default: + properties.push(_.merge(_.pick(value, ['arrival_format', 'data_type', 'is_deleted']), {expr: newExpr, name: newKey})) + } + }); + } + flatten(_.get(dataSchema, 'properties'), undefined, undefined) + return properties + } + + /** + * Get all fields of a dataset merging schema fields, transformations and denorm fields + * + * @param data_schema + * @param transformations_config + * @param denorm_config + * @returns Promise[]> + */ + getAllFields = async (data_schema: any, transformations_config: any, denorm_config: any) : Promise[]> => { + const instance = this; + let dataFields = instance.flattenSchema(data_schema); + if(denorm_config.denorm_fields) { + _.map(denorm_config.denorm_fields, async (denormField) => { + const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); + const properties = instance.flattenSchema(denormDataset.data_schema); + const transformProps = _.map(properties, (prop) => { + _.set(prop, 'name', _.join([denormField.denorm_out_field, prop.name], '.')); + _.set(prop, 'expr', _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + return prop; + }); + dataFields.push(...transformProps); + }) + } + if(transformations_config) { + _.map(transformations_config, async (tf) => { + dataFields.push({ + expr: "$." + tf.field_key, + name: tf.field_key, + data_type: tf.transformation_function.datatype + }) + }) + } + dataFields.push(rawIngestionSpecDefaults.synctsField) + _.remove(dataFields, {is_deleted: true}) // Delete all the excluded fields + return dataFields; + } +} + +class TableGenerator extends BaseTableGenerator { + + getDruidIngestionSpec = async (dataset: Record, datasourceRef: string) => { + + const { data_schema, denorm_config, dataset_config, router_config, transformations_config} = dataset + const allFields = await this.getAllFields(data_schema, transformations_config, denorm_config); + return { + "type": "kafka", + "spec": { + "dataSchema": { + "dataSource": datasourceRef, + "dimensionsSpec": { "dimensions": this.getDimensions(allFields, dataset_config.keys_config.timestamp_key, dataset_config.keys_config.partition_key) }, + "timestampSpec": { "column": dataset_config.keys_config.timestamp_key, "format": "auto" }, + "metricsSpec": [], + "granularitySpec": rawIngestionSpecDefaults.granularitySpec + }, + "tuningConfig": rawIngestionSpecDefaults.tuningConfig, + "ioConfig": _.merge(rawIngestionSpecDefaults.ioConfig, { + "topic": router_config.topic, + "inputFormat": { + "flattenSpec": { + "fields": this.getFlattenSpec(allFields) + } + } + }) + } + } + } + + getDimensions = (allFields: Record[], timestampKey: string, partitionKey: string | undefined) => { + if(partitionKey) { // Move the partition column to the top of the dimensions + const partitionCol = _.remove(allFields, {name: partitionKey}) + if(partitionCol && _.size(partitionCol) > 0) { + allFields.unshift(partitionCol[0]) + } + } + const instance = this; + return _.union( + _.map(allFields, (field) => { + if(field.name !== timestampKey) { + return { + "type": instance.getType(field.data_type), + "name": field.name + } + } + }), + rawIngestionSpecDefaults.dimensions + ) + } + + getType = (data_type: string):string => { + switch (data_type) { + case "number": return "double"; + case "integer": return "long"; + case "object": return "json"; + case "boolean": return "string"; + case "array": return "json"; + case "string": return "string"; + default: return "auto"; + } + } + + getFlattenSpec = (allFields: Record) => { + return _.union( + _.map(allFields, (field) => { + return { + type: "path", + expr: field.expr, + name: field.name + } + }), + rawIngestionSpecDefaults.flattenSpec + ) + } +} + +export const tableGenerator = new TableGenerator(); \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts index 3dccb0a9..29a8584d 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts @@ -5,7 +5,6 @@ import spies from "chai-spies"; import httpStatus from "http-status"; import { describe, it } from 'mocha'; import _ from "lodash"; -import { apiId } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; @@ -48,7 +47,7 @@ describe("DATASET STATUS TRANSITION DELETE", () => { .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) @@ -69,7 +68,7 @@ describe("DATASET STATUS TRANSITION DELETE", () => { .end((err, res) => { res.should.have.status(httpStatus.NOT_FOUND); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) res.body.error.message.should.be.eq("Dataset not found to delete") diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 3e3e40bc..4623a0e3 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -5,7 +5,6 @@ import spies from "chai-spies"; import httpStatus from "http-status"; import { describe, it } from 'mocha'; import _ from "lodash"; -import { apiId, errorCode } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { commandHttpService } from "../../../connections/commandServiceConnection"; @@ -36,7 +35,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) @@ -57,7 +56,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { .end((err, res) => { res.should.have.status(httpStatus.NOT_FOUND); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) res.body.error.message.should.be.eq("Dataset not found to perform status transition to live") @@ -80,9 +79,8 @@ describe("DATASET STATUS TRANSITION LIVE", () => { .end((err, res) => { res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) res.body.error.message.should.be.eq("Failed to perform status transition on datasets") done(); }); @@ -99,7 +97,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { .end((err, res) => { res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.error.code.should.be.eq("DATASET_LIVE_FAILURE") res.body.error.message.should.be.eq("Failed to mark dataset Live as it is not in ready to publish state") diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index ba77c0a2..83173c68 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -5,7 +5,6 @@ import spies from "chai-spies"; import httpStatus from "http-status"; import { describe, it } from 'mocha'; import _ from "lodash"; -import { apiId } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; @@ -36,7 +35,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) @@ -57,7 +56,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { .end((err, res) => { res.should.have.status(httpStatus.NOT_FOUND); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) res.body.error.message.should.be.eq("Dataset not found to perform status transition to ready to publish") @@ -77,7 +76,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { .end((err, res) => { res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) res.body.error.message.should.be.eq("Failed to mark dataset Ready to publish as it not in draft state") @@ -98,7 +97,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { .end((err, res) => { res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) expect(res.body.error.message).to.match(/^#required must have(.+)/) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index ce47c13c..21d721d3 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -5,7 +5,6 @@ import spies from "chai-spies"; import httpStatus from "http-status"; import { describe, it } from 'mocha'; import _ from "lodash"; -import { apiId, errorCode } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { Dataset } from "../../../models/Dataset"; import { DatasetDraft } from "../../../models/DatasetDraft"; @@ -59,7 +58,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) @@ -101,7 +100,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) @@ -143,7 +142,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) @@ -164,7 +163,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { .end((err, res) => { res.should.have.status(httpStatus.NOT_FOUND); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) res.body.error.message.should.be.eq("Dataset not found to retire") @@ -184,7 +183,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { .end((err, res) => { res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) res.body.error.message.should.be.eq("Failed to Retire dataset as it is not in live state") @@ -210,7 +209,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { .end((err, res) => { res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) res.body.error.message.should.be.eq("Failed to retire dataset as it is used by other datasets") @@ -233,9 +232,8 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { .end((err, res) => { res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) res.body.error.message.should.be.eq("Failed to perform status transition on datasets") done(); }); @@ -270,9 +268,8 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { .end((err, res) => { res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) res.body.error.message.should.be.eq("Failed to perform status transition on datasets") done(); }); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts index ee589998..da2d4450 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts @@ -5,7 +5,6 @@ import spies from "chai-spies"; import httpStatus from "http-status"; import { describe, it } from 'mocha'; import _ from "lodash"; -import { apiId, errorCode } from "../../../controllers/DatasetStatusTransition/DatasetStatusTransition"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; @@ -31,7 +30,7 @@ describe("DATASET STATUS TRANSITION API", () => { .end((err, res) => { res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) expect(res.body.error.message).to.match(/^#properties\/request(.+)$/) @@ -51,9 +50,8 @@ describe("DATASET STATUS TRANSITION API", () => { .end((err, res) => { res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); + res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") - res.body.error.code.should.be.eq(errorCode) res.body.error.message.should.be.eq("Failed to perform status transition on datasets") done(); }); diff --git a/api-service/src/v2/types/ObsrvError.ts b/api-service/src/v2/types/ObsrvError.ts new file mode 100644 index 00000000..94fef6a2 --- /dev/null +++ b/api-service/src/v2/types/ObsrvError.ts @@ -0,0 +1,24 @@ +export class ObsrvError { + code: string; + message: string; + errCode: string; + statusCode: number; + data: any | undefined; + datasetId: string; + err: Error | undefined; + + constructor(datasetId: string, code: string, message: string, errorCode: string, statusCode: number, err?: Error, data?: any) { + this.datasetId = datasetId; + this.code = code; + this.message = message; + this.errCode = errorCode; + this.statusCode = statusCode; + this.data = data; + this.err = err; + } + +} + +export const obsrvError = (datasetId: string, code: string, message: string, errorCode: string, statusCode: number, err?: Error, data?: any) => { + return new ObsrvError(datasetId, code, message, errorCode, statusCode, err, data) +} \ No newline at end of file From aade0b328dde22a79fd01d6ba034c03632985cac Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 11 Jul 2024 20:00:13 +0530 Subject: [PATCH 020/235] #OBS-I115: Dataset Transition API refactoring --- api-service/src/v2/configs/IngestionConfig.ts | 5 +- api-service/src/v2/services/DatasetService.ts | 48 +++-- api-service/src/v2/services/TableGenerator.ts | 195 +++++++++++++++--- 3 files changed, 207 insertions(+), 41 deletions(-) diff --git a/api-service/src/v2/configs/IngestionConfig.ts b/api-service/src/v2/configs/IngestionConfig.ts index c566be57..8826b264 100644 --- a/api-service/src/v2/configs/IngestionConfig.ts +++ b/api-service/src/v2/configs/IngestionConfig.ts @@ -51,6 +51,7 @@ export const rawIngestionSpecDefaults = { "name": "obsrv_meta.syncts", "arrival_format": "text", "data_type": "date", + "type": "text", "expr": "$.obsrv_meta.syncts" }, "dimensions": [ @@ -66,12 +67,12 @@ export const rawIngestionSpecDefaults = { "flattenSpec": [ { "type": "path", - "expr": "$.obsrv_meta.source.['connector']", + "expr": "$.obsrv_meta.source.connector", "name": "obsrv.meta.source.connector" }, { "type": "path", - "expr": "$.obsrv_meta.source.['connectorInstance']", + "expr": "$.obsrv_meta.source.connectorInstance", "name": "obsrv.meta.source.id" } ] diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 18a608a8..514b83d3 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -243,10 +243,21 @@ class DatasetService { publishDataset = async (draftDataset: Record) => { + const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() try { await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } , transaction}) - await this.generateDataSouce(draftDataset, transaction) + if(indexingConfig.olap_store_enabled) { + await this.createDruidDataSource(draftDataset, transaction); + } + if(indexingConfig.lakehouse_enabled) { + const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id"], true); + if(liveDataset) { + await this.updateHudiDataSource(draftDataset, transaction) + } else { + await this.createHudiDataSource(draftDataset, transaction) + } + } await transaction.commit() await executeCommand(draftDataset.id, "PUBLISH_DATASET"); } catch(err:any) { @@ -256,25 +267,38 @@ class DatasetService { } - generateDataSouce = async (draftDataset: Record, transaction: Transaction) => { + createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { - const indexingConfig = draftDataset.dataset_config.indexing_config; - if(indexingConfig.olap_store_enabled) { - const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); - const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, draftDatasource.datasource_ref); - _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) - } - if(indexingConfig.lakehouse_enabled) { + const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); + const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); + const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); + _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + await DatasourceDraft.create(draftDatasource, {transaction}) + } - } + createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + + const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); + const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); + const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); + _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + await DatasourceDraft.create(draftDatasource, {transaction}) + } + + updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + + const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); + const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); + const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); + _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + await DatasourceDraft.create(draftDatasource, {transaction}) } createDraftDatasource = (draftDataset: Record, type: string) : Record => { const datasource = _.join([draftDataset.dataset_id,"events"], "_") return { - id: datasource, + id: _.join([datasource,type], '_'), datasource: draftDataset.dataset_id, dataset_id: draftDataset.dataset_id, datasource_ref: datasource, diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts index 7772b1b3..fea99b3d 100644 --- a/api-service/src/v2/services/TableGenerator.ts +++ b/api-service/src/v2/services/TableGenerator.ts @@ -10,7 +10,7 @@ class BaseTableGenerator { * @param dataSchema * @returns properties Record[] */ - flattenSchema = (dataSchema: Record) : Record[] => { + flattenSchema = (dataSchema: Record, type: string) : Record[] => { let properties: Record[] = [] const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { @@ -22,17 +22,17 @@ class BaseTableGenerator { flatten(_.get(value, 'properties'), newKey, newExpr); break; case 'array': - if(_.get(value, 'items.type') == 'object' && _.get(value, 'items.properties')) { + if(type === "druid" && _.get(value, 'items.type') == 'object' && _.get(value, 'items.properties')) { _.mapKeys(_.get(value, 'items.properties'), function(value, childKey) { const objChildKey = _.join([newKey, childKey], '.') - properties.push(_.merge(_.pick(value, ['arrival_format', 'is_deleted']), {expr: _.join([newExpr,"[*].['",childKey,"']"], ''), name: objChildKey, data_type: 'array'})) + properties.push(_.merge(_.pick(value, ['type', 'arrival_format', 'is_deleted']), {expr: _.join([newExpr,"[*].['",childKey,"']"], ''), name: objChildKey, data_type: 'array'})) }) } else { - properties.push(_.merge(_.pick(value, ['arrival_format', 'data_type', 'is_deleted']), {expr: newExpr, name: newKey})) + properties.push(_.merge(_.pick(value, ['arrival_format', 'data_type', 'is_deleted']), {expr: newExpr+'[*]', name: newKey, type: _.get(value, 'items.type')})) } break; default: - properties.push(_.merge(_.pick(value, ['arrival_format', 'data_type', 'is_deleted']), {expr: newExpr, name: newKey})) + properties.push(_.merge(_.pick(value, ['type', 'arrival_format', 'data_type', 'is_deleted']), {expr: newExpr, name: newKey})) } }); } @@ -48,13 +48,15 @@ class BaseTableGenerator { * @param denorm_config * @returns Promise[]> */ - getAllFields = async (data_schema: any, transformations_config: any, denorm_config: any) : Promise[]> => { + getAllFields = async (dataset: Record, type: string) : Promise[]> => { + + const { data_schema, denorm_config, transformations_config} = dataset const instance = this; - let dataFields = instance.flattenSchema(data_schema); + let dataFields = instance.flattenSchema(data_schema, type); if(denorm_config.denorm_fields) { _.map(denorm_config.denorm_fields, async (denormField) => { const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); - const properties = instance.flattenSchema(denormDataset.data_schema); + const properties = instance.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { _.set(prop, 'name', _.join([denormField.denorm_out_field, prop.name], '.')); _.set(prop, 'expr', _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); @@ -68,7 +70,9 @@ class BaseTableGenerator { dataFields.push({ expr: "$." + tf.field_key, name: tf.field_key, - data_type: tf.transformation_function.datatype + data_type: tf.transformation_function.datatype, + arrival_format: tf.transformation_function.datatype, + type: tf.transformation_function.datatype }) }) } @@ -80,17 +84,16 @@ class BaseTableGenerator { class TableGenerator extends BaseTableGenerator { - getDruidIngestionSpec = async (dataset: Record, datasourceRef: string) => { + getDruidIngestionSpec = (dataset: Record, allFields: Record[], datasourceRef: string) => { - const { data_schema, denorm_config, dataset_config, router_config, transformations_config} = dataset - const allFields = await this.getAllFields(data_schema, transformations_config, denorm_config); + const { dataset_config, router_config } = dataset return { "type": "kafka", "spec": { "dataSchema": { "dataSource": datasourceRef, - "dimensionsSpec": { "dimensions": this.getDimensions(allFields, dataset_config.keys_config.timestamp_key, dataset_config.keys_config.partition_key) }, - "timestampSpec": { "column": dataset_config.keys_config.timestamp_key, "format": "auto" }, + "dimensionsSpec": { "dimensions": this.getDruidDimensions(allFields, this.getTimestampKey(dataset), dataset_config.keys_config.partition_key) }, + "timestampSpec": { "column": this.getTimestampKey(dataset), "format": "auto" }, "metricsSpec": [], "granularitySpec": rawIngestionSpecDefaults.granularitySpec }, @@ -99,7 +102,7 @@ class TableGenerator extends BaseTableGenerator { "topic": router_config.topic, "inputFormat": { "flattenSpec": { - "fields": this.getFlattenSpec(allFields) + "fields": this.getDruidFlattenSpec(allFields) } } }) @@ -107,28 +110,29 @@ class TableGenerator extends BaseTableGenerator { } } - getDimensions = (allFields: Record[], timestampKey: string, partitionKey: string | undefined) => { + getDruidDimensions = (allFields: Record[], timestampKey: string, partitionKey: string | undefined) => { + + const dataFields = _.cloneDeep(allFields); if(partitionKey) { // Move the partition column to the top of the dimensions - const partitionCol = _.remove(allFields, {name: partitionKey}) + const partitionCol = _.remove(dataFields, {name: partitionKey}) if(partitionCol && _.size(partitionCol) > 0) { - allFields.unshift(partitionCol[0]) + dataFields.unshift(partitionCol[0]) } } + _.remove(dataFields, {name: timestampKey}) const instance = this; return _.union( - _.map(allFields, (field) => { - if(field.name !== timestampKey) { - return { - "type": instance.getType(field.data_type), - "name": field.name - } + _.map(dataFields, (field) => { + return { + "type": instance.getDruidDimensionType(field.data_type), + "name": field.name } }), rawIngestionSpecDefaults.dimensions ) } - getType = (data_type: string):string => { + getDruidDimensionType = (data_type: string):string => { switch (data_type) { case "number": return "double"; case "integer": return "long"; @@ -140,7 +144,7 @@ class TableGenerator extends BaseTableGenerator { } } - getFlattenSpec = (allFields: Record) => { + getDruidFlattenSpec = (allFields: Record) => { return _.union( _.map(allFields, (field) => { return { @@ -152,6 +156,143 @@ class TableGenerator extends BaseTableGenerator { rawIngestionSpecDefaults.flattenSpec ) } + + getHudiIngestionSpecForCreate = (dataset: Record, allFields: Record[], datasourceRef: string) => { + + const primaryKey = this.getPrimaryKey(dataset); + const partitionKey = this.getHudiPartitionKey(dataset); + const timestampKey = this.getTimestampKey(dataset); + return { + dataset: dataset.dataset_id, + schema: { + table: datasourceRef, + partitionColumn: partitionKey, + timestampColumn: timestampKey, + primaryKey: primaryKey, + columnSpec: this.getHudiColumnSpec(allFields, primaryKey, partitionKey, timestampKey) + }, + inputFormat: { + type: "json", + flattenSpec: { + fields: this.getHudiFields(allFields) + } + } + } + } + + getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string) : Record[] => { + + const instance = this; + const dataFields = _.cloneDeep(allFields); + _.remove(dataFields, {name: primaryKey}) + _.remove(dataFields, {name: partitionKey}) + _.remove(dataFields, {name: timestampKey}) + let index = 1; + const transformFields = _.map(dataFields, (field) => { + return { + "type": instance.getHudiColumnType(field), + "name": field.name, + "index": index++ + } + }) + _.each(rawIngestionSpecDefaults.dimensions, (field) => { + transformFields.push({ + "type": field.type, + "name": field.name, + "index": index++ + }) + }) + return transformFields; + } + + getHudiColumnType = (field: Record) : string => { + if(field.data_type === 'array' && field.arrival_format !== 'array') { + return "array"; + } + if(field.data_type === 'array' && field.arrival_format === 'array') { + switch(field.type) { + case "string": + return "array" + case "number": + return "array" + case "integer": + return "array" + case "boolean": + return "array" + default: + return "array" + } + } + switch(field.arrival_format) { + case "text": + return "string" + case "number": + switch(field.data_type) { + case "integer": + return "int" + case "epoch": + return "epoch" + case "bigdecimal": + return "bigdecimal" + case "float": + return "float" + case "long": + return "long" + default: + return "double" + } + case "integer": + return "int" + case "boolean": + return "boolean" + default: + return "string" + } + } + + getHudiFields = (allFields: Record[]) : Record[] => { + + return _.union( + _.map(allFields, (field) => { + return { + type: "path", + expr: _.replace(field.expr, /[\[\]'\*]/g, ""), + name: field.name + } + }), + rawIngestionSpecDefaults.flattenSpec + ) + } + + getPrimaryKey = (dataset: Record) : string => { + return dataset.dataset_config.keys_config.data_key; + } + + getHudiPartitionKey = (dataset: Record) : string => { + return dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; + } + + getTimestampKey = (dataset: Record) : string => { + return dataset.dataset_config.keys_config.timestamp_key; + } } -export const tableGenerator = new TableGenerator(); \ No newline at end of file +export const tableGenerator = new TableGenerator(); + +const schema = '{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"userid":{"type":"string","arrival_format":"text","data_type":"string"},"block":{"type":"string","arrival_format":"text","data_type":"string"},"cluster":{"type":"string","arrival_format":"text","data_type":"string"},"schooludisecode":{"type":"string","arrival_format":"text","data_type":"string"},"schoolname":{"type":"string","arrival_format":"text","data_type":"string"},"usertype":{"type":"string","arrival_format":"text","data_type":"string"},"usersubtype":{"type":"string","arrival_format":"text","data_type":"string"},"board":{"type":"string","arrival_format":"text","data_type":"string"},"rootorgid":{"type":"string","arrival_format":"text","data_type":"string"},"orgname":{"type":"string","arrival_format":"text","data_type":"string"},"subject":{"type":"array","items":{"type":"string"},"arrival_format":"array","data_type":"array"},"language":{"type":"array","items":{"type":"string"},"arrival_format":"array","data_type":"array"},"grade":{"type":"array","items":{"type":"string"},"arrival_format":"array","data_type":"array"},"framework":{"type":"string","arrival_format":"text","data_type":"string"},"medium":{"type":"array","items":{"type":"string"},"arrival_format":"array","data_type":"array"},"district":{"type":"string","arrival_format":"text","data_type":"string"},"profileusertypes":{"type":"array","items":{"type":"object","properties":{"type":{"type":"string","arrival_format":"text","data_type":"string"},"subType":{"type":"string","arrival_format":"text","data_type":"string"}},"additionalProperties":false},"arrival_format":"array","data_type":"array"}},"additionalProperties":false}'; +const dataSchema = JSON.parse(schema); +const allFields = tableGenerator.flattenSchema(dataSchema, "druid") +const dataset = { + dataset_id: "ny_trip_data", + router_config: { + topic: "ny_trip_data" + }, + dataset_config: { + keys_config: { + data_key: "userid", + timestamp_key: "grade", + partition_key: "block", + } + } +} +console.log(JSON.stringify(tableGenerator.getDruidIngestionSpec(dataset, allFields, "ny_trip_data_events"))) \ No newline at end of file From 405c2922149f8a92f6e731ae0a68a3908b72bd5c Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 11 Jul 2024 21:52:54 +0530 Subject: [PATCH 021/235] #OBS-I115: Dataset Transition API refactoring --- api-service/src/v2/services/DatasetService.ts | 12 +++--- api-service/src/v2/services/TableGenerator.ts | 38 +++++-------------- 2 files changed, 16 insertions(+), 34 deletions(-) diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 514b83d3..82b6aab6 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -135,7 +135,7 @@ class DatasetService { return await this.getDraftDataset(datasetId); } - getTransformationCategory = (section: string):string => { + private getTransformationCategory = (section: string):string => { switch(section) { case "pii": @@ -225,7 +225,7 @@ class DatasetService { } } - deleteDruidSupervisors = async (dataset: Record) => { + private deleteDruidSupervisors = async (dataset: Record) => { try { if (dataset.type !== DatasetType.master) { @@ -267,7 +267,7 @@ class DatasetService { } - createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { + private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); @@ -276,7 +276,7 @@ class DatasetService { await DatasourceDraft.create(draftDatasource, {transaction}) } - createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); @@ -285,7 +285,7 @@ class DatasetService { await DatasourceDraft.create(draftDatasource, {transaction}) } - updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); @@ -294,7 +294,7 @@ class DatasetService { await DatasourceDraft.create(draftDatasource, {transaction}) } - createDraftDatasource = (draftDataset: Record, type: string) : Record => { + private createDraftDatasource = (draftDataset: Record, type: string) : Record => { const datasource = _.join([draftDataset.dataset_id,"events"], "_") return { diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts index fea99b3d..233b8c47 100644 --- a/api-service/src/v2/services/TableGenerator.ts +++ b/api-service/src/v2/services/TableGenerator.ts @@ -110,7 +110,7 @@ class TableGenerator extends BaseTableGenerator { } } - getDruidDimensions = (allFields: Record[], timestampKey: string, partitionKey: string | undefined) => { + private getDruidDimensions = (allFields: Record[], timestampKey: string, partitionKey: string | undefined) => { const dataFields = _.cloneDeep(allFields); if(partitionKey) { // Move the partition column to the top of the dimensions @@ -132,7 +132,7 @@ class TableGenerator extends BaseTableGenerator { ) } - getDruidDimensionType = (data_type: string):string => { + private getDruidDimensionType = (data_type: string):string => { switch (data_type) { case "number": return "double"; case "integer": return "long"; @@ -144,7 +144,7 @@ class TableGenerator extends BaseTableGenerator { } } - getDruidFlattenSpec = (allFields: Record) => { + private getDruidFlattenSpec = (allFields: Record) => { return _.union( _.map(allFields, (field) => { return { @@ -180,7 +180,7 @@ class TableGenerator extends BaseTableGenerator { } } - getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string) : Record[] => { + private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string) : Record[] => { const instance = this; const dataFields = _.cloneDeep(allFields); @@ -205,7 +205,7 @@ class TableGenerator extends BaseTableGenerator { return transformFields; } - getHudiColumnType = (field: Record) : string => { + private getHudiColumnType = (field: Record) : string => { if(field.data_type === 'array' && field.arrival_format !== 'array') { return "array"; } @@ -250,7 +250,7 @@ class TableGenerator extends BaseTableGenerator { } } - getHudiFields = (allFields: Record[]) : Record[] => { + private getHudiFields = (allFields: Record[]) : Record[] => { return _.union( _.map(allFields, (field) => { @@ -264,35 +264,17 @@ class TableGenerator extends BaseTableGenerator { ) } - getPrimaryKey = (dataset: Record) : string => { + private getPrimaryKey = (dataset: Record) : string => { return dataset.dataset_config.keys_config.data_key; } - getHudiPartitionKey = (dataset: Record) : string => { + private getHudiPartitionKey = (dataset: Record) : string => { return dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; } - getTimestampKey = (dataset: Record) : string => { + private getTimestampKey = (dataset: Record) : string => { return dataset.dataset_config.keys_config.timestamp_key; } } -export const tableGenerator = new TableGenerator(); - -const schema = '{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"object","properties":{"userid":{"type":"string","arrival_format":"text","data_type":"string"},"block":{"type":"string","arrival_format":"text","data_type":"string"},"cluster":{"type":"string","arrival_format":"text","data_type":"string"},"schooludisecode":{"type":"string","arrival_format":"text","data_type":"string"},"schoolname":{"type":"string","arrival_format":"text","data_type":"string"},"usertype":{"type":"string","arrival_format":"text","data_type":"string"},"usersubtype":{"type":"string","arrival_format":"text","data_type":"string"},"board":{"type":"string","arrival_format":"text","data_type":"string"},"rootorgid":{"type":"string","arrival_format":"text","data_type":"string"},"orgname":{"type":"string","arrival_format":"text","data_type":"string"},"subject":{"type":"array","items":{"type":"string"},"arrival_format":"array","data_type":"array"},"language":{"type":"array","items":{"type":"string"},"arrival_format":"array","data_type":"array"},"grade":{"type":"array","items":{"type":"string"},"arrival_format":"array","data_type":"array"},"framework":{"type":"string","arrival_format":"text","data_type":"string"},"medium":{"type":"array","items":{"type":"string"},"arrival_format":"array","data_type":"array"},"district":{"type":"string","arrival_format":"text","data_type":"string"},"profileusertypes":{"type":"array","items":{"type":"object","properties":{"type":{"type":"string","arrival_format":"text","data_type":"string"},"subType":{"type":"string","arrival_format":"text","data_type":"string"}},"additionalProperties":false},"arrival_format":"array","data_type":"array"}},"additionalProperties":false}'; -const dataSchema = JSON.parse(schema); -const allFields = tableGenerator.flattenSchema(dataSchema, "druid") -const dataset = { - dataset_id: "ny_trip_data", - router_config: { - topic: "ny_trip_data" - }, - dataset_config: { - keys_config: { - data_key: "userid", - timestamp_key: "grade", - partition_key: "block", - } - } -} -console.log(JSON.stringify(tableGenerator.getDruidIngestionSpec(dataset, allFields, "ny_trip_data_events"))) \ No newline at end of file +export const tableGenerator = new TableGenerator(); \ No newline at end of file From 1f3dd927d76ef5dea58282aeb438f7074aebd483 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 11 Jul 2024 22:14:04 +0530 Subject: [PATCH 022/235] #OBS-I115: Remove unnecessary field fields_set --- api-service/src/v2/models/Dataset.ts | 4 ---- api-service/src/v2/models/DatasetDraft.ts | 4 ---- 2 files changed, 8 deletions(-) diff --git a/api-service/src/v2/models/Dataset.ts b/api-service/src/v2/models/Dataset.ts index 2989f309..72d636b7 100644 --- a/api-service/src/v2/models/Dataset.ts +++ b/api-service/src/v2/models/Dataset.ts @@ -68,10 +68,6 @@ export const Dataset = sequelize.define("datasets", { version: { type: DataTypes.NUMBER }, - fields_set: { - type: DataTypes.JSON, - defaultValue: {} - }, sample_data: { type: DataTypes.JSON, defaultValue: {} diff --git a/api-service/src/v2/models/DatasetDraft.ts b/api-service/src/v2/models/DatasetDraft.ts index d049abf5..264dbce3 100644 --- a/api-service/src/v2/models/DatasetDraft.ts +++ b/api-service/src/v2/models/DatasetDraft.ts @@ -98,10 +98,6 @@ export const DatasetDraft = sequelize.define("datasets_draft", { type: DataTypes.STRING, defaultValue: "v2" }, - fields_set: { - type: DataTypes.JSON, - defaultValue: {} - }, sample_data: { type: DataTypes.JSON, defaultValue: {} From fb51360e11c9cdb05ba963089477d41d761c96bc Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 11 Jul 2024 22:42:55 +0530 Subject: [PATCH 023/235] #OBS-I115: Dataset publish API - update the index of hudi spec properly for publish to handle schema evolution --- api-service/src/v2/services/DatasetService.ts | 4 +++- api-service/src/v2/services/TableGenerator.ts | 21 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 82b6aab6..43bbcdf0 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -289,7 +289,9 @@ class DatasetService { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); - const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); + const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") + const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record + const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource.ingestion_spec, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts index 233b8c47..6ac3fb05 100644 --- a/api-service/src/v2/services/TableGenerator.ts +++ b/api-service/src/v2/services/TableGenerator.ts @@ -180,6 +180,27 @@ class TableGenerator extends BaseTableGenerator { } } + getHudiIngestionSpecForUpdate = (dataset: Record, existingHudiSpec: Record, allFields: Record[], datasourceRef: string) => { + + let newHudiSpec = this.getHudiIngestionSpecForCreate(dataset, allFields, datasourceRef) + + const newColumnSpec = newHudiSpec.schema.columnSpec; + let oldColumnSpec = existingHudiSpec.schema.columnSpec; + let currIndex = _.get(_.maxBy(oldColumnSpec, 'index'), 'index') as unknown as number + const newColumns = _.differenceBy(newColumnSpec, oldColumnSpec, 'name'); + if(_.size(newColumns) > 0) { + _.each(newColumns, (col) => { + oldColumnSpec.push({ + "type": col.type, + "name": col.name, + "index": currIndex++ + }) + }) + } + _.set(newHudiSpec, 'schema.columnSpec', oldColumnSpec) + return newHudiSpec; + } + private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string) : Record[] => { const instance = this; From 1b5f65ddcf4be7b82ad47963d5aedca707c399dd Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 11 Jul 2024 23:20:48 +0530 Subject: [PATCH 024/235] #OBS-I116: Dataset create and status transition api code fix --- api-service/src/app.ts | 3 +-- api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts | 2 +- .../DatasetStatusTransition/DatasetStatusTransition.ts | 4 ++-- .../DatasetStatusTransition/ReadyToPublishSchema.json | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 2dbbd887..66cbaa4d 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -25,7 +25,6 @@ app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.use(errorHandler) -app.use(obsrvErrorHandler); app.set("queryServices", services); loadExtensions(app) @@ -35,7 +34,7 @@ loadExtensions(app) app.use("/", router); app.use("/", metricsRouter); app.use("*", ResponseHandler.routeNotFound); - app.use(ResponseHandler.errorResponse); + app.use(obsrvErrorHandler); app.listen(config.api_port, () => { console.log(`listening on port ${config.api_port}`); diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index 6b648305..0906098e 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -32,7 +32,7 @@ const validateRequest = async (req: Request) => { const datasetCreate = async (req: Request, res: Response) => { - validateRequest(req) + await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index a9592233..948a7772 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -34,7 +34,7 @@ const validateRequest = (req: Request, datasetId: any) => { const validateDataset = (dataset: any, datasetId: any, action: string) => { if (_.isEmpty(dataset)) { - throw obsrvError(datasetId, datasetNotFound, `Dataset not found for dataset: ${dataset.id}`, "NOT_FOUND", 404) + throw obsrvError(datasetId, datasetNotFound, `Dataset not found for dataset: ${datasetId}`, "NOT_FOUND", 404) } if (dataset.api_version !== "v2" && _.includes(["ReadyToPublish", "Live"], action)) { @@ -54,7 +54,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); validateRequest(req, dataset_id); - const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type"]) + const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) validateDataset(dataset, dataset_id, status); switch(status) { diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index c5e7785e..10f0ca91 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -18,7 +18,7 @@ }, "type": { "type": "string", - "enum": ["dataset", "master-dataset"] + "enum": ["event", "transaction", "master"] }, "name": { "type": "string", @@ -305,7 +305,7 @@ } }, "sample_data": { - "type": "string" + "type": "object" }, "transformations_config": { "type": "array", From 0a39770cc7f32061aaa26418799ddc6527fb61c3 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 12 Jul 2024 16:22:00 +0530 Subject: [PATCH 025/235] #OBS-I116: Dataset Create api test case fixes --- .../DatasetManagement/DatasetCreate/DatasetCreate.spec.ts | 4 ++-- .../tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index 0aec1b47..b1cf2db2 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -74,7 +74,7 @@ describe("DATASET CREATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq(fixture.status) res.body.params.msgid.should.be.eq(fixture.msgid) - res.body.error.message.should.be.eq("Duplicate denorm key found") + res.body.error.message.should.be.eq("Duplicate denorm output fields found.") res.body.error.code.should.be.eq("DATASET_DUPLICATE_DENORM_KEY") done(); }); @@ -113,7 +113,7 @@ describe("DATASET CREATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset already exists") + res.body.error.message.should.be.eq("Dataset Already exists with id:sb-ddd") res.body.error.code.should.be.eq("DATASET_EXISTS") done(); }); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 97ec0282..6accc269 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -181,7 +181,7 @@ describe("DATASET READ API", () => { res.body.should.be.a("object") res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Dataset with the given dataset_id not found") + res.body.error.message.should.be.eq("Dataset with the given dataset_id:sb-telemetry not found") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); From 0cd3eb3ec3528e33e2c3159c9db699898385dc68 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 12 Jul 2024 16:37:41 +0530 Subject: [PATCH 026/235] #OBS-I116: Dataset update extraction config api test case fixes --- .../DatasetUpdate/DatasetExtraction.spec.ts | 18 ++++- .../DatasetUpdate/DatasetTags.spec.ts | 74 +------------------ .../DatasetUpdate/Fixtures.ts | 64 ++++------------ 3 files changed, 31 insertions(+), 125 deletions(-) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts index 4d7c8131..923f2c5e 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts @@ -22,7 +22,7 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { it("Success: Dataset extraction configs updated if it is a batch event", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, api_version: "v2", type: "event" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { it("Success: Dataset extraction configs updated with default values if it is not batch event", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, api_version: "v2", type: "event" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -57,7 +57,19 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { chai .request(app) .patch("/v2/datasets/update") - .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, extraction_config: { "is_batch_event": false } } }) + .send({ + ...requestStructure, request: { + dataset_id: "telemetry", version_key: validVersionKey, + "extraction_config": { + "is_batch_event": false, + "extraction_key": "events", + "dedup_config": { + "drop_duplicates": true, + "dedup_key": "id" + } + } + } + }) .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts index 90bb6c67..07471544 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts @@ -22,7 +22,7 @@ describe("DATASET TAGS UPDATE", () => { it("Success: Dataset tags successfully added", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type: "dataset", version_key: validVersionKey, denorm_config: { denorm_fields: [] } + id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, denorm_config: { denorm_fields: [] }, api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET TAGS UPDATE", () => { it("Success: Dataset tags successfully removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type: "dataset", version_key: validVersionKey, tags: ["tag1", "tag2"] + id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, tags: ["tag1", "tag2"], api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -71,74 +71,4 @@ describe("DATASET TAGS UPDATE", () => { }); }); - it("Success: When payload contains same tags to be added or removed", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", type: "dataset", version_key: validVersionKey, tags: ["tag1", "tag2"] - }) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_SAME_TAGS_ADD) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Failure: When tags provided to add already exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag3", "tag1"] - }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset tags already exist") - res.body.error.code.should.be.eq("DATASET_TAGS_EXISTS") - done(); - }); - }); - - it("Failure: When tags provided to delete does not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, tags: ["tag5"] - }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset tags do not exist to remove") - res.body.error.code.should.be.eq("DATASET_TAGS_DO_NOT_EXIST") - done(); - }); - }); - }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index b5652fa4..4e7e9b10 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -25,11 +25,8 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": [ - "tag1", - "tag2" - ], - "action": "add" + "values": "tag1", + "action": "upsert" }] } }, @@ -40,10 +37,7 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": [ - "tag1", - "tag2" - ], + "values": "tag1", "action": "remove" }] } @@ -98,7 +92,7 @@ export const TestInputsForDatasetUpdate = { "mode": "Strict", "metadata": {} }, - "action": "add" + "action": "upsert" }] } }, @@ -287,7 +281,7 @@ export const TestInputsForDatasetUpdate = { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, - "action": "add" + "action": "upsert" }, { "values": { @@ -306,7 +300,7 @@ export const TestInputsForDatasetUpdate = { "mode": "Strict", "metadata": {} }, - "action": "add" + "action": "upsert" }, { "values": { @@ -334,18 +328,12 @@ export const TestInputsForDatasetUpdate = { }, "tags": [ { - "values": [ - "tag1", - "tag2" - ], + "values": "tag1", "action": "remove" }, { - "values": [ - "tag3", - "tag4" - ], - "action": "add" + "values": "tag3", + "action": "upsert" } ] } @@ -362,44 +350,20 @@ export const TestInputsForDatasetUpdate = { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, - "action": "add" + "action": "upsert" }, { "values": { "denorm_key": "actor.id", "denorm_out_field": "userdata" }, - "action": "add" + "action": "upsert" } ] } } }, - - DATASET_UPDATE_WITH_SAME_TAGS_ADD: { - ...requestStructure, request: { - "dataset_id": "telemetry", - "version_key": validVersionKey, - "name": "sb-telemetry", - "tags": [ - { - "values": [ - "tag1", - "tag1" - ], - "action": "remove" - }, - { - "values": [ - "tag4", - "tag4" - ], - "action": "add" - } - ] - } - }, - + DATASET_UPDATE_WITH_SAME_DENORM_REMOVE: { ...requestStructure, request: { "dataset_id": "telemetry", @@ -441,7 +405,7 @@ export const TestInputsForDatasetUpdate = { "mode": "Strict", "metadata": {} }, - "action": "add" + "action": "upsert" }, { "values": { @@ -450,7 +414,7 @@ export const TestInputsForDatasetUpdate = { "mode": "Strict", "metadata": {} }, - "action": "add" + "action": "upsert" }, { "values": { From 9bfaf595f8c88c9f6c20fe2e30fd71dedde29811 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sun, 14 Jul 2024 23:08:49 +0530 Subject: [PATCH 027/235] #OBS-I116: Dataset druid ingestion spec generation fix --- api-service/src/v2/services/TableGenerator.ts | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts index 6ac3fb05..069d1dca 100644 --- a/api-service/src/v2/services/TableGenerator.ts +++ b/api-service/src/v2/services/TableGenerator.ts @@ -48,13 +48,13 @@ class BaseTableGenerator { * @param denorm_config * @returns Promise[]> */ - getAllFields = async (dataset: Record, type: string) : Promise[]> => { + getAllFields = async (dataset: Record, type: string): Promise[]> => { - const { data_schema, denorm_config, transformations_config} = dataset + const { data_schema, denorm_config, transformations_config } = dataset const instance = this; let dataFields = instance.flattenSchema(data_schema, type); - if(denorm_config.denorm_fields) { - _.map(denorm_config.denorm_fields, async (denormField) => { + if (denorm_config.denorm_fields) { + for (const denormField of denorm_config.denorm_fields) { const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); const properties = instance.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { @@ -63,21 +63,21 @@ class BaseTableGenerator { return prop; }); dataFields.push(...transformProps); - }) + } } - if(transformations_config) { - _.map(transformations_config, async (tf) => { - dataFields.push({ - expr: "$." + tf.field_key, - name: tf.field_key, - data_type: tf.transformation_function.datatype, - arrival_format: tf.transformation_function.datatype, - type: tf.transformation_function.datatype - }) - }) + if (!_.isEmpty(transformations_config)) { + const transformationFields = _.map(transformations_config, (tf) => ({ + expr: "$." + tf.field_key, + name: tf.field_key, + data_type: tf.transformation_function.datatype, + arrival_format: tf.transformation_function.datatype, + type: tf.transformation_function.datatype + })) + const originalFields = _.differenceBy(dataFields, transformationFields, "name") + dataFields = _.concat(originalFields, transformationFields) } dataFields.push(rawIngestionSpecDefaults.synctsField) - _.remove(dataFields, {is_deleted: true}) // Delete all the excluded fields + _.remove(dataFields, { is_deleted: true }) // Delete all the excluded fields return dataFields; } } From bb5bf381e05e9192ea3896461d34b30452c06686 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sun, 14 Jul 2024 23:09:43 +0530 Subject: [PATCH 028/235] #OBS-I116: express version upgraded --- api-service/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/package.json b/api-service/package.json index d34a4325..4359e729 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -30,7 +30,7 @@ "body-parser": "^1.20.2", "compression": "^1.7.4", "dateformat": "2.0.0", - "express": "^4.18.2", + "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", "http-status": "^1.5.3", "kafka-node": "^5.0.0", From 781c8afd2bd6493c39cb73ccfe4b90cf14f7864a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 15 Jul 2024 18:07:35 +0530 Subject: [PATCH 029/235] #OBS-I116: Dataset create api fixes --- api-service/package.json | 2 +- api-service/src/app.ts | 3 +-- api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/api-service/package.json b/api-service/package.json index d34a4325..4359e729 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -30,7 +30,7 @@ "body-parser": "^1.20.2", "compression": "^1.7.4", "dateformat": "2.0.0", - "express": "^4.18.2", + "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", "http-status": "^1.5.3", "kafka-node": "^5.0.0", diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 2dbbd887..66cbaa4d 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -25,7 +25,6 @@ app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.use(errorHandler) -app.use(obsrvErrorHandler); app.set("queryServices", services); loadExtensions(app) @@ -35,7 +34,7 @@ loadExtensions(app) app.use("/", router); app.use("/", metricsRouter); app.use("*", ResponseHandler.routeNotFound); - app.use(ResponseHandler.errorResponse); + app.use(obsrvErrorHandler); app.listen(config.api_port, () => { console.log(`listening on port ${config.api_port}`); diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts index 6b648305..0906098e 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts @@ -32,7 +32,7 @@ const validateRequest = async (req: Request) => { const datasetCreate = async (req: Request, res: Response) => { - validateRequest(req) + await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); From 00ac3cb98052e44a43c01faf719d416ef8f66572 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 16 Jul 2024 09:30:50 +0530 Subject: [PATCH 030/235] #OBS-I116: Dataset ingestion spec generation fix --- api-service/src/v2/services/TableGenerator.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts index 069d1dca..838538b7 100644 --- a/api-service/src/v2/services/TableGenerator.ts +++ b/api-service/src/v2/services/TableGenerator.ts @@ -53,7 +53,7 @@ class BaseTableGenerator { const { data_schema, denorm_config, transformations_config } = dataset const instance = this; let dataFields = instance.flattenSchema(data_schema, type); - if (denorm_config.denorm_fields) { + if (!_.isEmpty(denorm_config.denorm_fields)) { for (const denormField of denorm_config.denorm_fields) { const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); const properties = instance.flattenSchema(denormDataset.data_schema, type); From a12fd980a967957deda4ddb130205f120fde0128 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 16 Jul 2024 09:38:29 +0530 Subject: [PATCH 031/235] #OBS-I126: updated swagger documentation --- .../swagger-doc/v2_updated_doc_openapi.yml | 2033 +++++++++++++++++ 1 file changed, 2033 insertions(+) create mode 100644 api-service/swagger-doc/v2_updated_doc_openapi.yml diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml new file mode 100644 index 00000000..8bdde0a2 --- /dev/null +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -0,0 +1,2033 @@ +openapi: 3.0.0 +info: + title: V2 docs + version: 1.0.0 +servers: + - url: localhost:// + - url: http://localhost:3007 +tags: + - name: Dataset management api's +paths: + /v2/datasets/create: + post: + tags: + - Dataset management api's + summary: API to create a new dataset. + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.create + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + type: event + name: sb-telemetry + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_config: + drop_duplicates: true + dedup_key: mid + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + mid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + arrival_format: number + data_type: epoch + eid: + type: string + arrival_format: text + data_type: string + additionalProperties: true + denorm_config: + denorm_fields: + - denorm_key: eid + denorm_out_field: userdata + dataset_id: master-telemetry + transformations_config: + - field_key: email + transformation_function: + type: mask + expr: mid + datatype: string + category: pii + mode: Strict + tags: + - tag1 + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Dataset created successfullly' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-15T18:44:08+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 276c042c-0f23-4b26-9b10-6fe48bbc2d3a + responseCode: OK + result: + id: telemetry_record-t4 + version_key: '1721049248930' + example-1: + summary: 'Success: Master dataset created successfully' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T08:36:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 845076be-d9e7-4246-bb8e-07ae0ce59d1e + responseCode: OK + result: + id: telemetry_record-master + version_key: '1721099200603' + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T08:39:00+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a07de860-dcbc-4ff6-822e-34b47635c8a3 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'dataset_id' + '409': + description: Conflict + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Master dataset already exists' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T08:37:28+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 138b796b-1b68-481a-a59d-1cb695c1adc9 + responseCode: CONFLICT + result: {} + error: + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-master + example-1: + summary: 'Failure: Dataset already exists' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T08:38:05+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf62693c-3aa4-42ce-a5ea-4bde340740f5 + responseCode: CONFLICT + result: {} + error: + code: DATASET_EXISTS + message: Dataset Already exists with id:telemetry_record-t4 + /v2/files/generate-url: + post: + tags: + - Dataset management api's + summary: API to generate presigned url's + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.files.generate-url + ver: v1 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: write + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Generate put url' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T08:26:19+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5306f309-4a15-458e-89e2-29d8ac0835d4 + responseCode: OK + result: + - filePath: >- + test-connector/api-service/user_uploads/telemetry_10d595.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject + - filePath: >- + test-connector/api-service/user_uploads/school_data_33109a.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject + example-1: + summary: 'Success: Generate get url' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T09:31:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 009c0b2d-8acd-40b0-a807-bbacf9242771 + responseCode: OK + result: + - filePath: test-connector/api-service/user_uploads/telemetry.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject + - filePath: >- + test-connector/api-service/user_uploads/school_data.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: limit exceeds' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T08:33:04+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d3a606ca-47d0-4746-95a1-c8692e749959 + responseCode: BAD_REQUEST + error: + code: FILES_URL_GENERATION_LIMIT_EXCEED + message: 'Pre-signed URL generation failed: limit exceeded.' + trace: '' + example-1: + summary: 'Failure: Invalid request' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T09:31:10+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: c3e9da1c-09f3-4a3b-84ec-a19efc68b856 + responseCode: BAD_REQUEST + error: + code: FILES_GENERATE_URL_INPUT_INVALID + message: >- + #properties/request/properties/access/enum must be equal + to one of the allowed values + trace: '' + /dataset/v1/dataschema: + post: + tags: + - Dataset management api's + summary: Data schema generator + requestBody: + content: + application/json: + schema: + type: object + example: + data: + - tripID: b97b0dbd-1463-4a42-99ff-211fc389464c + VendorID: '1' + tpep_pickup_datetime: '2024-02-12 00:22:30' + tpep_dropoff_datetime: '2023-09-14 00:40:50' + passenger_count: '2' + trip_distance: '2.80' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '249' + DOLocationID: '162' + payment_type: '2' + primary_passenger: + email: Raquel.Kunde@gmail.com + mobile: 310-255-4865 x1413 + fare_details: + fare_amount: '13.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '14.8' + congestion_surcharge: '' + - tripID: a0520d95-1ae2-4d94-a6d2-0e35857e2b3c + VendorID: '1' + tpep_pickup_datetime: '2023-07-01 00:47:00' + tpep_dropoff_datetime: '2024-01-25 00:52:18' + passenger_count: '1' + trip_distance: '1.00' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '140' + payment_type: '2' + primary_passenger: + email: Willy15@gmail.com + mobile: 474-817-2801 x633 + fare_details: + fare_amount: '6' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '7.3' + congestion_surcharge: '' + - tripID: 6248522a-ffbb-4d73-8b89-75a0e5310f25 + VendorID: '1' + tpep_pickup_datetime: '2023-03-19 00:55:39' + tpep_dropoff_datetime: '2023-03-16 01:03:06' + passenger_count: '4' + trip_distance: '1.90' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '140' + DOLocationID: '162' + payment_type: '2' + primary_passenger: + email: Agustina74@yahoo.com + mobile: 1-533-609-5857 x24749 + fare_details: + fare_amount: '8' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.3' + congestion_surcharge: '' + - tripID: 3502a658-1bf8-4d62-a180-f1560d088d36 + VendorID: '2' + tpep_pickup_datetime: '2023-06-02 00:28:37' + tpep_dropoff_datetime: '2024-01-26 00:31:37' + passenger_count: '1' + trip_distance: '.76' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '142' + DOLocationID: '239' + payment_type: '1' + primary_passenger: + email: Shemar97@hotmail.com + mobile: (738) 409-8443 x5839 + fare_details: + fare_amount: '4.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '7.8' + congestion_surcharge: '' + - tripID: 245a42b8-be3f-49a7-b82b-755a60c0fe90 + VendorID: '2' + tpep_pickup_datetime: '2023-03-16 00:33:19' + tpep_dropoff_datetime: '2023-06-17 00:46:44' + passenger_count: '1' + trip_distance: '3.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '239' + DOLocationID: '68' + payment_type: '1' + primary_passenger: + email: Alvis.Kshlerin77@hotmail.com + mobile: 1-487-796-9469 x057 + fare_details: + fare_amount: '13.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '16.8' + congestion_surcharge: '' + - tripID: 4e351ad9-b554-49fe-bdeb-351ced4a5fbc + VendorID: '2' + tpep_pickup_datetime: '2023-05-11 00:59:05' + tpep_dropoff_datetime: '2023-06-21 01:19:25' + passenger_count: '2' + trip_distance: '3.80' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '246' + DOLocationID: '43' + payment_type: '2' + primary_passenger: + email: Nicole.White@hotmail.com + mobile: 687-578-1535 x50831 + fare_details: + fare_amount: '16.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '17.8' + congestion_surcharge: '' + - tripID: 9f24c946-4f7d-4e2b-b0f0-34c11b61b97e + VendorID: '2' + tpep_pickup_datetime: '2023-03-04 00:11:27' + tpep_dropoff_datetime: '2024-01-14 00:46:29' + passenger_count: '5' + trip_distance: '21.42' + RatecodeID: '2' + store_and_fwd_flag: 'N' + PULocationID: '132' + DOLocationID: '87' + payment_type: '1' + primary_passenger: + email: Talia.Denesik@hotmail.com + mobile: 727-537-1685 x107 + fare_details: + fare_amount: '52' + extra: '0' + mta_tax: '0.5' + tip_amount: '11.71' + tolls_amount: '5.76' + improvement_surcharge: '0.3' + total_amount: '70.27' + congestion_surcharge: '' + - tripID: 4622df23-a8f1-482f-9310-76d6fb772b9a + VendorID: '2' + tpep_pickup_datetime: '2023-04-02 00:19:27' + tpep_dropoff_datetime: '2023-03-06 01:03:21' + passenger_count: '1' + trip_distance: '10.84' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '114' + DOLocationID: '198' + payment_type: '2' + primary_passenger: + email: Alice43@gmail.com + mobile: 579-402-1634 x87762 + fare_details: + fare_amount: '37.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '38.8' + congestion_surcharge: '' + - tripID: 8f8211bf-9693-4b36-817f-d9c1c7253691 + VendorID: '1' + tpep_pickup_datetime: '2023-05-12 00:11:49' + tpep_dropoff_datetime: '2023-02-28 00:23:07' + passenger_count: '2' + trip_distance: '3.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '161' + DOLocationID: '41' + payment_type: '1' + primary_passenger: + email: Mike88@gmail.com + mobile: (501) 617-2020 x4697 + fare_details: + fare_amount: '12.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.5' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '16.3' + congestion_surcharge: '' + - tripID: 6a102595-7db4-4184-bd80-ce249b784f0b + VendorID: '1' + tpep_pickup_datetime: '2023-12-02 00:38:41' + tpep_dropoff_datetime: '2024-02-01 01:08:52' + passenger_count: '4' + trip_distance: '3.20' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '161' + DOLocationID: '249' + payment_type: '2' + primary_passenger: + email: Vella.Armstrong71@gmail.com + mobile: 978.794.7934 x32048 + fare_details: + fare_amount: '18.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '19.8' + congestion_surcharge: '' + - tripID: ceced196-7da9-4bac-83ad-fe0129098574 + VendorID: '2' + tpep_pickup_datetime: '2024-02-20 00:14:19' + tpep_dropoff_datetime: '2023-11-22 00:17:03' + passenger_count: '2' + trip_distance: '.78' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '236' + payment_type: '1' + primary_passenger: + email: Keven_Kihn@hotmail.com + mobile: 1-323-467-0737 x980 + fare_details: + fare_amount: '4.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.16' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '6.96' + congestion_surcharge: '' + - tripID: 23a467f1-de9b-474f-a60a-3d103ebeba04 + VendorID: '2' + tpep_pickup_datetime: '2023-04-03 00:20:57' + tpep_dropoff_datetime: '2023-11-12 00:30:01' + passenger_count: '1' + trip_distance: '1.71' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '142' + payment_type: '2' + primary_passenger: + email: Kiera.Kling@hotmail.com + mobile: 704-373-7606 x93095 + fare_details: + fare_amount: '8.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.8' + congestion_surcharge: '' + - tripID: 15c7f4d7-fa74-41ee-98d5-7c00f30bb366 + VendorID: '2' + tpep_pickup_datetime: '2023-07-25 00:36:13' + tpep_dropoff_datetime: '2023-08-19 00:46:08' + passenger_count: '2' + trip_distance: '2.28' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '43' + DOLocationID: '151' + payment_type: '1' + primary_passenger: + email: Laura_Lind99@hotmail.com + mobile: (846) 893-6673 x69711 + fare_details: + fare_amount: '9.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.16' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '12.96' + congestion_surcharge: '' + - tripID: 796dfa5d-e16d-49c9-b41c-f4223a8fedd9 + VendorID: '2' + tpep_pickup_datetime: '2023-08-14 00:56:36' + tpep_dropoff_datetime: '2023-07-08 01:04:49' + passenger_count: '2' + trip_distance: '1.57' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '238' + DOLocationID: '41' + payment_type: '1' + primary_passenger: + email: Melvin68@yahoo.com + mobile: 1-436-319-7744 x1743 + fare_details: + fare_amount: '8' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.86' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '11.16' + congestion_surcharge: '' + - tripID: b8b56888-8bb4-4981-b669-2a56d563b25f + VendorID: '2' + tpep_pickup_datetime: '2023-06-23 00:27:41' + tpep_dropoff_datetime: '2024-02-12 00:42:43' + passenger_count: '1' + trip_distance: '2.80' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '113' + DOLocationID: '229' + payment_type: '2' + primary_passenger: + email: Mauricio8@gmail.com + mobile: 1-298-756-7810 x0828 + fare_details: + fare_amount: '12' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '13.3' + congestion_surcharge: '' + - tripID: 4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2 + VendorID: '2' + tpep_pickup_datetime: '2023-09-30 00:46:37' + tpep_dropoff_datetime: '2023-04-07 00:53:37' + passenger_count: '1' + trip_distance: '1.18' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '141' + DOLocationID: '262' + payment_type: '1' + primary_passenger: + email: Bailee.Roob80@gmail.com + mobile: 753.766.7597 x58905 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.66' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.96' + congestion_surcharge: '' + - tripID: dc5fb067-ee76-4c74-8d53-913518d7de3c + VendorID: '1' + tpep_pickup_datetime: '2023-09-07 00:21:23' + tpep_dropoff_datetime: '2023-08-14 00:26:07' + passenger_count: '2' + trip_distance: '.40' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '233' + DOLocationID: '233' + payment_type: '2' + primary_passenger: + email: Francisco70@yahoo.com + mobile: 1-591-565-3358 x04096 + fare_details: + fare_amount: '4.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '5.8' + congestion_surcharge: '' + - tripID: d1008fa4-1017-43e5-a7cd-7fcc235d82e5 + VendorID: '1' + tpep_pickup_datetime: '2023-08-13 00:49:45' + tpep_dropoff_datetime: '2023-08-30 00:56:01' + passenger_count: '1' + trip_distance: '1.00' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '113' + DOLocationID: '79' + payment_type: '2' + primary_passenger: + email: Charity_Jacobs58@gmail.com + mobile: 1-713-793-4442 x6226 + fare_details: + fare_amount: '6' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '7.3' + congestion_surcharge: '' + - tripID: a4d907fa-cad9-41fa-a25f-cd3fac14d0d9 + VendorID: '1' + tpep_pickup_datetime: '2023-10-27 00:56:42' + tpep_dropoff_datetime: '2023-03-23 01:26:09' + passenger_count: '4' + trip_distance: '6.00' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '79' + DOLocationID: '75' + payment_type: '1' + primary_passenger: + email: Bridie61@yahoo.com + mobile: (632) 512-8857 + fare_details: + fare_amount: '24' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '5.05' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '30.35' + congestion_surcharge: '' + - tripID: 28c84ea3-4129-4114-b0ee-7bb39cb9613f + VendorID: '2' + tpep_pickup_datetime: '2024-03-07 00:10:50' + tpep_dropoff_datetime: '2023-06-10 00:17:46' + passenger_count: '5' + trip_distance: '1.32' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '42' + DOLocationID: '41' + payment_type: '2' + primary_passenger: + email: Vickie.Franey93@gmail.com + mobile: 874.663.4243 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + - tripID: 1218ae38-7474-4a4b-9343-e3d0fe6da9c6 + VendorID: '2' + tpep_pickup_datetime: '2023-07-18 00:32:36' + tpep_dropoff_datetime: '2023-06-16 00:54:08' + passenger_count: '3' + trip_distance: '3.65' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '43' + DOLocationID: '224' + payment_type: '2' + primary_passenger: + email: Alexandro.Prohaska40@hotmail.com + mobile: (413) 378-0852 x100 + fare_details: + fare_amount: '15.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '16.8' + congestion_surcharge: '' + - tripID: 0a5fc580-5a28-4153-b37f-a2ecc4ebc777 + VendorID: '1' + tpep_pickup_datetime: '2023-05-06 00:32:24' + tpep_dropoff_datetime: '2023-11-29 00:33:21' + passenger_count: '0' + trip_distance: '5.30' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '50' + DOLocationID: '50' + payment_type: '1' + primary_passenger: + email: Veda_Stamm@hotmail.com + mobile: 596-555-2936 x826 + fare_details: + fare_amount: '2.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0.75' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '4.55' + congestion_surcharge: '' + - tripID: 89856e17-912a-40e6-8379-f1d75a066878 + VendorID: '1' + tpep_pickup_datetime: '2023-05-30 00:36:37' + tpep_dropoff_datetime: '2024-02-14 00:42:40' + passenger_count: '1' + trip_distance: '1.40' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '50' + DOLocationID: '239' + payment_type: '1' + primary_passenger: + email: Jed60@hotmail.com + mobile: (761) 368-4573 + fare_details: + fare_amount: '6.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.55' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.35' + congestion_surcharge: '' + - tripID: fece3c74-e914-46c6-8b84-88470d9c7b3a + VendorID: '1' + tpep_pickup_datetime: '2023-09-14 00:44:29' + tpep_dropoff_datetime: '2023-03-22 00:48:05' + passenger_count: '3' + trip_distance: '.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '238' + DOLocationID: '238' + payment_type: '2' + primary_passenger: + email: Juwan_Schimmel@yahoo.com + mobile: 1-725-659-7644 + fare_details: + fare_amount: '4.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '5.8' + congestion_surcharge: '' + - tripID: 9142ffc4-db10-4faf-ad6a-b15595e02408 + VendorID: '1' + tpep_pickup_datetime: '2024-02-14 00:52:53' + tpep_dropoff_datetime: '2023-09-29 01:27:03' + passenger_count: '2' + trip_distance: '14.40' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '238' + DOLocationID: '98' + payment_type: '2' + primary_passenger: + email: Pinkie13@hotmail.com + mobile: (991) 905-4690 + fare_details: + fare_amount: '42' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '5.76' + improvement_surcharge: '0.3' + total_amount: '49.06' + congestion_surcharge: '' + - tripID: ac62b151-e94c-473f-b461-c25f7035e8ac + VendorID: '2' + tpep_pickup_datetime: '2023-08-01 00:28:12' + tpep_dropoff_datetime: '2023-03-08 00:34:55' + passenger_count: '1' + trip_distance: '1.43' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '14' + DOLocationID: '228' + payment_type: '2' + primary_passenger: + email: Esteban_Ondricka@yahoo.com + mobile: (697) 411-7922 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + - tripID: 707e720d-afdd-4af9-9469-02fd5052ed9f + VendorID: '2' + tpep_pickup_datetime: '2023-10-10 00:55:16' + tpep_dropoff_datetime: '2023-03-25 01:01:05' + passenger_count: '1' + trip_distance: '1.00' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '11' + DOLocationID: '22' + payment_type: '2' + primary_passenger: + email: Kristin.Baumbach-Ferry@hotmail.com + mobile: 964-998-1199 + fare_details: + fare_amount: '6' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '7.3' + congestion_surcharge: '' + - tripID: e75c5819-6f52-4c57-a6ea-85b79935821b + VendorID: '1' + tpep_pickup_datetime: '2023-07-11 00:29:10' + tpep_dropoff_datetime: '2023-08-10 01:23:09' + passenger_count: '2' + trip_distance: '11.00' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '181' + DOLocationID: '239' + payment_type: '1' + primary_passenger: + email: Dorothea.Kulas@gmail.com + mobile: 354.367.7954 + fare_details: + fare_amount: '40' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '10.33' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '51.63' + congestion_surcharge: '' + - tripID: d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d + VendorID: '2' + tpep_pickup_datetime: '2024-01-12 00:20:30' + tpep_dropoff_datetime: '2023-11-09 00:36:07' + passenger_count: '1' + trip_distance: '1.50' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '161' + DOLocationID: '229' + payment_type: '2' + primary_passenger: + email: Jazmyn_Corwin@gmail.com + mobile: 642.790.7928 x83713 + fare_details: + fare_amount: '10.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '11.8' + congestion_surcharge: '' + - tripID: 6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba + VendorID: '2' + tpep_pickup_datetime: '2024-02-04 00:42:50' + tpep_dropoff_datetime: '2024-03-03 00:51:48' + passenger_count: '1' + trip_distance: '.72' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '162' + DOLocationID: '43' + payment_type: '2' + primary_passenger: + email: Mazie18@hotmail.com + mobile: 1-641-639-4170 x249 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + - tripID: 18578bdf-e169-4d3d-ae08-b6320bb04e29 + VendorID: '2' + tpep_pickup_datetime: '2023-04-19 00:53:39' + tpep_dropoff_datetime: '2023-05-12 00:51:48' + passenger_count: '1' + trip_distance: '2.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '43' + DOLocationID: '137' + payment_type: '1' + primary_passenger: + email: Maud.Collins@gmail.com + mobile: 214.847.9872 + fare_details: + fare_amount: '13' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.14' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '16.44' + congestion_surcharge: '' + - tripID: b44c85e8-ebd2-481f-a9d5-52124dec673a + VendorID: '1' + tpep_pickup_datetime: '2023-02-25 00:16:33' + tpep_dropoff_datetime: '2023-09-01 00:23:16' + passenger_count: '1' + trip_distance: '1.50' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '237' + payment_type: '1' + primary_passenger: + email: Domenick_Pagac@yahoo.com + mobile: 742.839.7610 x189 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.65' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.95' + congestion_surcharge: '' + - tripID: dc355724-1226-4dbf-ace7-f5c34e39b7e7 + VendorID: '1' + tpep_pickup_datetime: '2023-07-23 00:24:43' + tpep_dropoff_datetime: '2023-06-26 00:30:13' + passenger_count: '1' + trip_distance: '1.00' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '237' + DOLocationID: '140' + payment_type: '2' + primary_passenger: + email: Kathryne1@yahoo.com + mobile: (900) 624-9537 + fare_details: + fare_amount: '6' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '7.3' + congestion_surcharge: '' + - tripID: e65bc848-a894-4fb7-b889-dd7d812ab793 + VendorID: '1' + tpep_pickup_datetime: '2023-12-01 00:34:18' + tpep_dropoff_datetime: '2023-10-11 00:38:47' + passenger_count: '1' + trip_distance: '.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '140' + DOLocationID: '237' + payment_type: '1' + primary_passenger: + email: Hilton.Jacobi@gmail.com + mobile: 1-637-451-2136 + fare_details: + fare_amount: '5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.25' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '7.55' + congestion_surcharge: '' + - tripID: 32bf8dee-cd36-4c9e-befb-be32256653e0 + VendorID: '1' + tpep_pickup_datetime: '2024-02-01 00:39:55' + tpep_dropoff_datetime: '2023-03-31 00:51:01' + passenger_count: '1' + trip_distance: '1.90' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '237' + DOLocationID: '238' + payment_type: '1' + primary_passenger: + email: Ole.Lindgren@hotmail.com + mobile: (728) 501-8337 x72810 + fare_details: + fare_amount: '9.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '3.2' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '14' + congestion_surcharge: '' + - tripID: 1e7bc9a0-ccc6-4855-8650-a46e8695c1a2 + VendorID: '1' + tpep_pickup_datetime: '2023-08-24 00:52:47' + tpep_dropoff_datetime: '2023-12-12 00:59:52' + passenger_count: '1' + trip_distance: '1.90' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '238' + DOLocationID: '237' + payment_type: '1' + primary_passenger: + email: Jazmin.Pollich-Little81@gmail.com + mobile: 517-815-1765 + fare_details: + fare_amount: '8' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.85' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '11.15' + congestion_surcharge: '' + - tripID: b81e5976-6581-4cad-914d-b1bf3a007c7b + VendorID: '1' + tpep_pickup_datetime: '2023-08-24 00:24:43' + tpep_dropoff_datetime: '2023-05-19 00:37:30' + passenger_count: '1' + trip_distance: '1.30' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '162' + DOLocationID: '107' + payment_type: '1' + primary_passenger: + email: Jake_Heidenreich52@gmail.com + mobile: 300-566-1457 x9243 + fare_details: + fare_amount: '9.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.15' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '12.95' + congestion_surcharge: '' + - tripID: beae8765-a627-4af7-ac5b-a5ba96f6d54f + VendorID: '1' + tpep_pickup_datetime: '2023-12-29 00:47:22' + tpep_dropoff_datetime: '2023-10-17 00:55:16' + passenger_count: '1' + trip_distance: '.80' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '107' + DOLocationID: '79' + payment_type: '1' + primary_passenger: + email: Mitchell.Green64@hotmail.com + mobile: 1-279-558-0312 x75465 + fare_details: + fare_amount: '6.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.55' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.35' + congestion_surcharge: '' + - tripID: a59b934c-583c-4fa3-a5cb-15041379e8da + VendorID: '1' + tpep_pickup_datetime: '2023-09-17 00:59:36' + tpep_dropoff_datetime: '2023-09-12 01:28:32' + passenger_count: '1' + trip_distance: '3.80' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '113' + DOLocationID: '142' + payment_type: '1' + primary_passenger: + email: Esther.Hintz@hotmail.com + mobile: 546-446-7171 x1903 + fare_details: + fare_amount: '19.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '4.15' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '24.95' + congestion_surcharge: '' + - tripID: 7ed9ad81-42cb-436e-93c5-d88d7245493f + VendorID: '1' + tpep_pickup_datetime: '2023-12-29 00:18:04' + tpep_dropoff_datetime: '2023-10-04 00:25:32' + passenger_count: '1' + trip_distance: '1.30' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '161' + DOLocationID: '137' + payment_type: '1' + primary_passenger: + email: Carolanne_Buckridge@gmail.com + mobile: (972) 507-5995 x716 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + - tripID: 4c325355-2689-482f-8107-ece7515ddc33 + VendorID: '1' + tpep_pickup_datetime: '2023-08-17 00:40:33' + tpep_dropoff_datetime: '2023-03-17 00:44:25' + passenger_count: '2' + trip_distance: '.40' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '164' + DOLocationID: '170' + payment_type: '4' + primary_passenger: + email: Willie.Zieme44@hotmail.com + mobile: 813.930.8291 x27037 + fare_details: + fare_amount: '4.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '5.8' + congestion_surcharge: '' + - tripID: 5e153af5-1e69-4acc-bd70-25b7d6e9856a + VendorID: '1' + tpep_pickup_datetime: '2023-09-23 00:59:13' + tpep_dropoff_datetime: '2023-10-21 01:01:44' + passenger_count: '6' + trip_distance: '.50' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '161' + DOLocationID: '161' + payment_type: '3' + primary_passenger: + email: Abe.Kassulke5@yahoo.com + mobile: 1-994-727-5845 x8326 + fare_details: + fare_amount: '4' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '5.3' + congestion_surcharge: '' + - tripID: eef4af05-c391-4a67-9ed9-0a6a2c23645c + VendorID: '1' + tpep_pickup_datetime: '2024-03-11 00:10:40' + tpep_dropoff_datetime: '2023-03-16 00:27:11' + passenger_count: '2' + trip_distance: '4.30' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '229' + DOLocationID: '223' + payment_type: '2' + primary_passenger: + email: Clifford.Fisher@yahoo.com + mobile: 1-934-478-8531 x33207 + fare_details: + fare_amount: '16' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '17.3' + congestion_surcharge: '' + - tripID: a790d399-dbbc-4483-805b-0fb001b6c6b7 + VendorID: '2' + tpep_pickup_datetime: '2023-06-27 00:25:03' + tpep_dropoff_datetime: '2024-03-07 01:02:07' + passenger_count: '6' + trip_distance: '3.58' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '211' + DOLocationID: '48' + payment_type: '2' + primary_passenger: + email: Carson.Hartmann-Lynch59@gmail.com + mobile: (856) 507-5220 x738 + fare_details: + fare_amount: '23' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '24.3' + congestion_surcharge: '' + - tripID: 10fb7db8-edee-4686-95e7-d07a17761fe9 + VendorID: '1' + tpep_pickup_datetime: '2024-02-17 00:26:54' + tpep_dropoff_datetime: '2023-08-06 00:49:47' + passenger_count: '2' + trip_distance: '2.10' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '158' + DOLocationID: '107' + payment_type: '2' + primary_passenger: + email: Brandyn.Powlowski43@yahoo.com + mobile: (972) 869-2846 + fare_details: + fare_amount: '14.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '15.8' + congestion_surcharge: '' + - tripID: b575d34f-8874-4b67-8918-293cccec8558 + VendorID: '2' + tpep_pickup_datetime: '2023-03-04 00:18:00' + tpep_dropoff_datetime: '2023-11-02 00:26:10' + passenger_count: '1' + trip_distance: '1.11' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '13' + DOLocationID: '231' + payment_type: '1' + primary_passenger: + email: Lelia71@hotmail.com + mobile: 298-955-0204 x145 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + - tripID: 4524ce02-c41d-4e83-82a1-e41b00d97dab + VendorID: '2' + tpep_pickup_datetime: '2024-01-25 00:33:25' + tpep_dropoff_datetime: '2023-10-29 01:07:18' + passenger_count: '1' + trip_distance: '5.63' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '113' + DOLocationID: '238' + payment_type: '1' + primary_passenger: + email: Magali_Mohr90@gmail.com + mobile: 1-339-745-7996 x126 + fare_details: + fare_amount: '24' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '5.06' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '30.36' + congestion_surcharge: '' + - tripID: 0a4d6094-f334-4e18-a1e9-1001820b6f89 + VendorID: '1' + tpep_pickup_datetime: '2024-01-26 00:11:00' + tpep_dropoff_datetime: '2023-08-01 00:15:28' + passenger_count: '1' + trip_distance: '.80' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '79' + DOLocationID: '79' + payment_type: '1' + primary_passenger: + email: Sydney.Sanford71@yahoo.com + mobile: 557.212.3262 x589 + fare_details: + fare_amount: '5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.26' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '7.56' + congestion_surcharge: '' + - tripID: 9245af5e-632e-4f21-9060-88522728ab73 + VendorID: '1' + tpep_pickup_datetime: '2023-09-07 00:17:57' + tpep_dropoff_datetime: '2023-09-14 00:27:43' + passenger_count: '1' + trip_distance: '3.70' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '107' + DOLocationID: '87' + payment_type: '1' + primary_passenger: + email: Deanna15@hotmail.com + mobile: 927-327-6309 x16689 + fare_details: + fare_amount: '13' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.85' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '17.15' + congestion_surcharge: '' + - tripID: 4dcce454-046c-4b67-bd80-ff773a3c3d96 + VendorID: '1' + tpep_pickup_datetime: '2023-08-16 00:35:11' + tpep_dropoff_datetime: '2023-05-07 00:58:40' + passenger_count: '1' + trip_distance: '5.40' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '261' + DOLocationID: '142' + payment_type: '1' + primary_passenger: + email: Jeramie33@yahoo.com + mobile: 757-419-8948 x7985 + fare_details: + fare_amount: '20.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '5.45' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '27.25' + congestion_surcharge: '' + - tripID: d5808aed-5e8e-447f-8c42-521f8472a24d + VendorID: '1' + tpep_pickup_datetime: '2024-01-23 00:12:48' + tpep_dropoff_datetime: '2023-07-16 00:23:48' + passenger_count: '1' + trip_distance: '1.80' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '211' + DOLocationID: '232' + payment_type: '2' + primary_passenger: + email: Jarrod_Bergstrom@hotmail.com + mobile: 388-916-2388 x911 + fare_details: + fare_amount: '9.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '10.8' + congestion_surcharge: '' + - tripID: faf99a1d-127f-432a-bdb9-39c91a205ec3 + VendorID: '1' + tpep_pickup_datetime: '2023-08-12 00:31:53' + tpep_dropoff_datetime: '2023-09-30 00:47:26' + passenger_count: '1' + trip_distance: '2.10' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '79' + DOLocationID: '164' + payment_type: '1' + primary_passenger: + email: America72@gmail.com + mobile: 528-291-8014 x700 + fare_details: + fare_amount: '11.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2.55' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '15.35' + congestion_surcharge: '' + - tripID: 21b8ea90-d3ab-4fe6-890e-2b8a911500e1 + VendorID: '1' + tpep_pickup_datetime: '2024-02-05 00:54:38' + tpep_dropoff_datetime: '2023-08-23 01:01:13' + passenger_count: '1' + trip_distance: '1.00' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '107' + DOLocationID: '170' + payment_type: '1' + primary_passenger: + email: Autumn_Kerluke@hotmail.com + mobile: 542.726.7058 + fare_details: + fare_amount: '6.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.8' + congestion_surcharge: '' + - tripID: 080446ad-cf54-429f-82e2-e54f4f8d4a9c + VendorID: '1' + tpep_pickup_datetime: '2023-07-16 00:00:58' + tpep_dropoff_datetime: '2023-10-29 00:06:38' + passenger_count: '1' + trip_distance: '.80' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '148' + DOLocationID: '79' + payment_type: '1' + primary_passenger: + email: Reanna_Conroy-Ratke@gmail.com + mobile: (565) 628-3638 + fare_details: + fare_amount: '5.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '2' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.8' + congestion_surcharge: '' + - tripID: 5ec8c804-9d6c-436b-be68-da3fa2ffcc8e + VendorID: '1' + tpep_pickup_datetime: '2023-03-08 00:14:58' + tpep_dropoff_datetime: '2024-02-09 00:24:33' + passenger_count: '1' + trip_distance: '2.70' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '4' + DOLocationID: '87' + payment_type: '1' + primary_passenger: + email: Randi0@yahoo.com + mobile: (248) 538-4300 x71383 + fare_details: + fare_amount: '11' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '13.3' + congestion_surcharge: '' + - tripID: 3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8 + VendorID: '1' + tpep_pickup_datetime: '2023-07-01 00:31:12' + tpep_dropoff_datetime: '2023-09-29 00:38:08' + passenger_count: '1' + trip_distance: '.70' + RatecodeID: '1' + store_and_fwd_flag: 'Y' + PULocationID: '148' + DOLocationID: '148' + payment_type: '1' + primary_passenger: + email: Ethyl74@yahoo.com + mobile: 292-960-2200 x317 + fare_details: + fare_amount: '6' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.8' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.1' + congestion_surcharge: '' + - tripID: dec9470a-6baa-492e-9220-d7ed626e2a99 + VendorID: '1' + tpep_pickup_datetime: '2024-02-22 00:43:21' + tpep_dropoff_datetime: '2024-03-02 00:50:49' + passenger_count: '1' + trip_distance: '1.10' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '79' + DOLocationID: '231' + payment_type: '1' + primary_passenger: + email: Makayla_Schneider18@hotmail.com + mobile: 1-885-537-0198 x47953 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.3' + congestion_surcharge: '' + - tripID: 904d9b1f-1f17-4cfc-8e63-8bf57257da5e + VendorID: '1' + tpep_pickup_datetime: '2023-12-29 00:54:14' + tpep_dropoff_datetime: '2023-05-03 01:02:32' + passenger_count: '1' + trip_distance: '1.50' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '231' + DOLocationID: '158' + payment_type: '1' + primary_passenger: + email: Cora.Grimes@yahoo.com + mobile: 843.706.7413 + fare_details: + fare_amount: '7.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.8' + congestion_surcharge: '' + - tripID: 57d8ed83-afa7-44cb-a0d4-723775602c34 + VendorID: '1' + tpep_pickup_datetime: '2024-02-02 00:16:34' + tpep_dropoff_datetime: '2023-09-11 00:24:45' + passenger_count: '2' + trip_distance: '1.20' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '142' + DOLocationID: '237' + payment_type: '2' + primary_passenger: + email: Tate.Bins@hotmail.com + mobile: (606) 682-9953 x671 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + - tripID: fff2f2c8-4a30-446c-90af-443dd493886c + VendorID: '1' + tpep_pickup_datetime: '2023-12-03 00:29:04' + tpep_dropoff_datetime: '2023-07-26 00:36:33' + passenger_count: '2' + trip_distance: '1.90' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '237' + DOLocationID: '262' + payment_type: '1' + primary_passenger: + email: Magnus.Jacobs@hotmail.com + mobile: (955) 449-9284 x00149 + fare_details: + fare_amount: '8' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.85' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '11.15' + congestion_surcharge: '' + - tripID: 004c4cc7-e809-4108-aa01-87b0ded794ad + VendorID: '1' + tpep_pickup_datetime: '2023-05-18 00:41:00' + tpep_dropoff_datetime: '2023-08-05 00:59:50' + passenger_count: '1' + trip_distance: '5.80' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '244' + payment_type: '2' + primary_passenger: + email: Palma24@gmail.com + mobile: 1-861-673-8247 x142 + fare_details: + fare_amount: '20' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '21.3' + congestion_surcharge: '' + - tripID: 1a01450d-537f-4f16-a2e9-f17021b077e9 + VendorID: '2' + tpep_pickup_datetime: '2024-01-31 00:20:53' + tpep_dropoff_datetime: '2023-12-21 00:40:21' + passenger_count: '2' + trip_distance: '5.70' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '164' + DOLocationID: '255' + payment_type: '2' + primary_passenger: + email: Jeffry18@gmail.com + mobile: (206) 748-5730 x64895 + fare_details: + fare_amount: '19.5' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '5.76' + improvement_surcharge: '0.3' + total_amount: '26.56' + congestion_surcharge: '' + - tripID: 5bccfd17-7813-43ca-9917-f2fe6ad49261 + VendorID: '2' + tpep_pickup_datetime: '2023-04-25 00:06:30' + tpep_dropoff_datetime: '2023-10-31 00:08:31' + passenger_count: '1' + trip_distance: '.46' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '239' + DOLocationID: '239' + payment_type: '1' + primary_passenger: + email: Leanne.Swaniawski@gmail.com + mobile: (787) 969-9302 x2684 + fare_details: + fare_amount: '4' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '1.59' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '6.89' + congestion_surcharge: '' + - tripID: e613b176-be28-4414-9a40-cff61e46db3b + VendorID: '2' + tpep_pickup_datetime: '2023-08-13 00:09:35' + tpep_dropoff_datetime: '2023-04-01 00:17:05' + passenger_count: '1' + trip_distance: '1.72' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '239' + DOLocationID: '236' + payment_type: '2' + primary_passenger: + email: Michel_Watsica63@yahoo.com + mobile: 793.751.7397 x892 + fare_details: + fare_amount: '8' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '9.3' + congestion_surcharge: '' + - tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + config: + dataset: generate-schema + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + example: + id: dataset.schema.identify + ver: v1 + ts: 1721100502574 + params: + status: SUCCESS + errmsg: '' + responseCode: OK + result: + schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + tripID: + type: string + suggestions: + - message: >- + The Property 'tripID' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.tripID + arrival_format: text + data_type: string + VendorID: + type: string + arrival_format: text + data_type: string + tpep_pickup_datetime: + type: string + suggestions: + - message: >- + The Property 'tpep_pickup_datetime' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.tpep_pickup_datetime + arrival_format: text + data_type: date-time + tpep_dropoff_datetime: + type: string + suggestions: + - message: >- + The Property 'tpep_dropoff_datetime' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.tpep_dropoff_datetime + arrival_format: text + data_type: date-time + passenger_count: + type: string + arrival_format: text + data_type: string + trip_distance: + type: string + arrival_format: text + data_type: string + RatecodeID: + type: string + arrival_format: text + data_type: string + store_and_fwd_flag: + type: string + arrival_format: text + data_type: string + PULocationID: + type: string + arrival_format: text + data_type: string + DOLocationID: + type: string + arrival_format: text + data_type: string + payment_type: + type: string + arrival_format: text + data_type: string + primary_passenger: + type: object + properties: + email: + type: string + arrival_format: text + data_type: string + mobile: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + fare_details: + type: object + properties: + fare_amount: + type: string + arrival_format: text + data_type: string + extra: + type: string + arrival_format: text + data_type: string + mta_tax: + type: string + arrival_format: text + data_type: string + tip_amount: + type: string + arrival_format: text + data_type: string + tolls_amount: + type: string + arrival_format: text + data_type: string + improvement_surcharge: + type: string + arrival_format: text + data_type: string + total_amount: + type: string + arrival_format: text + data_type: string + congestion_surcharge: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 98 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean \ No newline at end of file From edc2ca203d9bef841e55dbfbd47f89fef2144256 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 16 Jul 2024 09:39:54 +0530 Subject: [PATCH 032/235] #OBS-I126: updated postman collection --- .../updated_v2_collection.json | 806 ++++++++++++++++++ 1 file changed, 806 insertions(+) create mode 100644 api-service/postman-collection/updated_v2_collection.json diff --git a/api-service/postman-collection/updated_v2_collection.json b/api-service/postman-collection/updated_v2_collection.json new file mode 100644 index 00000000..ca750245 --- /dev/null +++ b/api-service/postman-collection/updated_v2_collection.json @@ -0,0 +1,806 @@ +{ + "info": { + "_postman_id": "04207e75-91d2-4896-a6ef-ea6a12c4018f", + "name": "V2 docs", + "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", + "_exporter_id": "33916975", + "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/33916975-04207e75-91d2-4896-a6ef-ea6a12c4018f?action=share&source=collection_link&creator=33916975" + }, + "item": [ + { + "name": "Dataset management api's", + "item": [ + { + "name": "Dataset create", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "response": [ + { + "name": "Success: Dataset created successfullly", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "287" + }, + { + "key": "ETag", + "value": "W/\"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ\"" + }, + { + "key": "Date", + "value": "Mon, 15 Jul 2024 13:14:09 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-15T18:44:08+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"276c042c-0f23-4b26-9b10-6fe48bbc2d3a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\"\n }\n}" + }, + { + "name": "Success: Master dataset created successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "291" + }, + { + "key": "ETag", + "value": "W/\"123-ZAXtVh5dFh84bZdu3SFBTQDhEHI\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:06:40 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:36:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"845076be-d9e7-4246-bb8e-07ae0ce59d1e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-master\",\n \"version_key\": \"1721099200603\"\n }\n}" + }, + { + "name": "Failure: Master dataset already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "337" + }, + { + "key": "ETag", + "value": "W/\"151-a7dJ9XBUyT3AXNxl1TPcraxMX08\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:07:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:37:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"138b796b-1b68-481a-a59d-1cb695c1adc9\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-master\"\n }\n}" + }, + { + "name": "Failure: Dataset already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "333" + }, + { + "key": "ETag", + "value": "W/\"14d-QLXc3MJG4hFiMbBoA6mXFDpSryQ\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:08:05 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:38:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf62693c-3aa4-42ce-a5ea-4bde340740f5\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-t4\"\n }\n}" + }, + { + "name": "Failure: Invalid request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "362" + }, + { + "key": "ETag", + "value": "W/\"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:09:00 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:39:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a07de860-dcbc-4ff6-822e-34b47635c8a3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + } + ] + }, + { + "name": "File generate url", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "response": [ + { + "name": "Success: Generate put url", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1344" + }, + { + "key": "ETag", + "value": "W/\"540-790rZel+H/rDwgvZRxvlUmZ8Gpc\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 02:56:19 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:26:19+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5306f309-4a15-458e-89e2-29d8ac0835d4\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry_10d595.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data_33109a.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject\"\n }\n ]\n}" + }, + { + "name": "Success: Generate get url", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1316" + }, + { + "key": "ETag", + "value": "W/\"524-iPflsPqFFV7NFaQCi2ODhQtbq/g\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 04:01:40 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"009c0b2d-8acd-40b0-a807-bbacf9242771\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject\"\n }\n ]\n}" + }, + { + "name": "Failure: limit exceeds", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "355" + }, + { + "key": "ETag", + "value": "W/\"163-9oQYJJEaBH3mJAnzDHXn2MxE848\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:03:04 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:33:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d3a606ca-47d0-4746-95a1-c8692e749959\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_URL_GENERATION_LIMIT_EXCEED\",\n \"message\": \"Pre-signed URL generation failed: limit exceeded.\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Invalid request", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"update\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "390" + }, + { + "key": "ETag", + "value": "W/\"186-KH9x0zC3+RqtWpa0tVT9XVg8agk\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 04:01:10 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"c3e9da1c-09f3-4a3b-84ec-a19efc68b856\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_GENERATE_URL_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/access/enum must be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Data schema generator", + "request": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json, text/plain, */*" + }, + { + "key": "Accept-Language", + "value": "en-GB,en" + }, + { + "key": "Cache-Control", + "value": "no-store" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ" + }, + { + "key": "Origin", + "value": "http://localhost:3001" + }, + { + "key": "Pragma", + "value": "no-store" + }, + { + "key": "Referer", + "value": "http://localhost:3001/console/dataset/new" + }, + { + "key": "Sec-Fetch-Dest", + "value": "empty" + }, + { + "key": "Sec-Fetch-Mode", + "value": "cors" + }, + { + "key": "Sec-Fetch-Site", + "value": "same-origin" + }, + { + "key": "Sec-GPC", + "value": "1" + }, + { + "key": "User-Agent", + "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" + }, + { + "key": "sec-ch-ua", + "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"" + }, + { + "key": "sec-ch-ua-mobile", + "value": "?0" + }, + { + "key": "sec-ch-ua-platform", + "value": "\"macOS\"" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3007/dataset/v1/dataschema" + }, + "response": [ + { + "name": "Success: Schema generated successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json, text/plain, */*", + "disabled": true + }, + { + "key": "Accept-Language", + "value": "en-GB,en", + "disabled": true + }, + { + "key": "Cache-Control", + "value": "no-store", + "disabled": true + }, + { + "key": "Connection", + "value": "keep-alive", + "disabled": true + }, + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", + "disabled": true + }, + { + "key": "Origin", + "value": "http://localhost:3001", + "disabled": true + }, + { + "key": "Pragma", + "value": "no-store", + "disabled": true + }, + { + "key": "Referer", + "value": "http://localhost:3001/console/dataset/new", + "disabled": true + }, + { + "key": "Sec-Fetch-Dest", + "value": "empty", + "disabled": true + }, + { + "key": "Sec-Fetch-Mode", + "value": "cors", + "disabled": true + }, + { + "key": "Sec-Fetch-Site", + "value": "same-origin", + "disabled": true + }, + { + "key": "Sec-GPC", + "value": "1", + "disabled": true + }, + { + "key": "User-Agent", + "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "disabled": true + }, + { + "key": "sec-ch-ua", + "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", + "disabled": true + }, + { + "key": "sec-ch-ua-mobile", + "value": "?0", + "disabled": true + }, + { + "key": "sec-ch-ua-platform", + "value": "\"macOS\"", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3007/dataset/v1/dataschema" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4414" + }, + { + "key": "ETag", + "value": "W/\"113e-ykaeY2EqBHdGqGLcr7K3WSs5fYo\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:28:22 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"dataset.schema.identify\",\n \"ver\": \"v1\",\n \"ts\": 1721100502574,\n \"params\": {\n \"status\": \"SUCCESS\",\n \"errmsg\": \"\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tripID' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tripID\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"VendorID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tpep_pickup_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_pickup_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_pickup_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"tpep_dropoff_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_dropoff_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_dropoff_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"passenger_count\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"trip_distance\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"RatecodeID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"store_and_fwd_flag\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"PULocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"DOLocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"payment_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primary_passenger\": {\n \"type\": \"object\",\n \"properties\": {\n \"email\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mobile\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"fare_details\": {\n \"type\": \"object\",\n \"properties\": {\n \"fare_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"extra\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mta_tax\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tip_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tolls_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"improvement_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"total_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"congestion_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 98,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + } + ] + } + ] + } + ] +} \ No newline at end of file From 367c8436fd4482b4911683c5f42dd33152094e68 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 16 Jul 2024 10:51:59 +0530 Subject: [PATCH 033/235] #OBS-I126: updated postman collection --- api-service/swagger-doc/v2_updated_doc_openapi.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml index 8bdde0a2..a0e9c254 100644 --- a/api-service/swagger-doc/v2_updated_doc_openapi.yml +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -3,15 +3,13 @@ info: title: V2 docs version: 1.0.0 servers: - - url: localhost:// + - url: localhost:3000 - url: http://localhost:3007 -tags: - - name: Dataset management api's paths: /v2/datasets/create: post: tags: - - Dataset management api's + - Dataset API's summary: API to create a new dataset. requestBody: content: @@ -174,7 +172,7 @@ paths: /v2/files/generate-url: post: tags: - - Dataset management api's + - Dataset API's summary: API to generate presigned url's requestBody: content: @@ -291,7 +289,7 @@ paths: /dataset/v1/dataschema: post: tags: - - Dataset management api's + - Dataset API's summary: Data schema generator requestBody: content: From efbc3aeeb9a82884f1520ebc4d0f718dd182465b Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 16 Jul 2024 18:02:43 +0530 Subject: [PATCH 034/235] #OBS-I116: fix: entry topic column in datasets model --- api-service/src/v2/models/Dataset.ts | 4 ++++ api-service/src/v2/models/DatasetDraft.ts | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/api-service/src/v2/models/Dataset.ts b/api-service/src/v2/models/Dataset.ts index 72d636b7..fa02cb10 100644 --- a/api-service/src/v2/models/Dataset.ts +++ b/api-service/src/v2/models/Dataset.ts @@ -71,6 +71,10 @@ export const Dataset = sequelize.define("datasets", { sample_data: { type: DataTypes.JSON, defaultValue: {} + }, + entry_topic: { + type: DataTypes.STRING, + allowNull: false } }, { tableName: "datasets", diff --git a/api-service/src/v2/models/DatasetDraft.ts b/api-service/src/v2/models/DatasetDraft.ts index 264dbce3..30f56766 100644 --- a/api-service/src/v2/models/DatasetDraft.ts +++ b/api-service/src/v2/models/DatasetDraft.ts @@ -99,8 +99,12 @@ export const DatasetDraft = sequelize.define("datasets_draft", { defaultValue: "v2" }, sample_data: { - type: DataTypes.JSON, - defaultValue: {} + type: DataTypes.JSON, + defaultValue: {} + }, + entry_topic: { + type: DataTypes.STRING, + allowNull: false } }, { timestamps: true, From aaaaca70f56af4207fefc7236476c024bace1916 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 17 Jul 2024 12:51:26 +0530 Subject: [PATCH 035/235] #OBS-I58 feat: Minio cloud store support - Added endpoint. as optional config to support the minio --- .../src/v2/services/CloudServices/AWSStorageService.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api-service/src/v2/services/CloudServices/AWSStorageService.ts b/api-service/src/v2/services/CloudServices/AWSStorageService.ts index ee6b3b0c..e538298f 100644 --- a/api-service/src/v2/services/CloudServices/AWSStorageService.ts +++ b/api-service/src/v2/services/CloudServices/AWSStorageService.ts @@ -15,7 +15,11 @@ export class AWSStorageService implements ICloudService { const region = _.get(config, "region") const accessKeyId = _.get(config, "identity") const secretAccessKey = _.get(config, "credential") - const configuration = { region, credentials: { accessKeyId, secretAccessKey } } + const endpoint = _.get(config, "endpoint") + const configuration: any = { region, credentials: { accessKeyId, secretAccessKey } } + if(endpoint) { + configuration.endpoint = endpoint; + } try { this.client = new S3Client(configuration); } From 6c86ba0672a63d833b39667c3796823889f107ea Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 17 Jul 2024 19:28:54 +0530 Subject: [PATCH 036/235] #OBS-I126: added dataset read, list, update api's documentation and updated collection --- .../updated_v2_collection.json | 1279 ++++++- .../swagger-doc/v2_updated_doc_openapi.yml | 2961 ++++++++++++++++- 2 files changed, 4218 insertions(+), 22 deletions(-) diff --git a/api-service/postman-collection/updated_v2_collection.json b/api-service/postman-collection/updated_v2_collection.json index ca750245..527881cd 100644 --- a/api-service/postman-collection/updated_v2_collection.json +++ b/api-service/postman-collection/updated_v2_collection.json @@ -8,7 +8,7 @@ }, "item": [ { - "name": "Dataset management api's", + "name": "Dataset api's", "item": [ { "name": "Dataset create", @@ -316,6 +316,120 @@ ], "cookie": [], "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:39:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a07de860-dcbc-4ff6-822e-34b47635c8a3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + }, + { + "name": "Success: Minimal request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_events\",\n \"type\":\"event\", //\"master\" for master dataset\n \"name\": \"sb-telemetry\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "284" + }, + { + "key": "ETag", + "value": "W/\"11c-kOoTX1K7Zbp+vsGfNEv87FBJOWg\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:44:59 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:14:59+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e207f4f-2be6-4a45-ab78-213bea272ae0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_events\",\n \"version_key\": \"1721133899306\"\n }\n}" + }, + { + "name": "Success: Dataset created successfully with all fields", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"/path/to/file1.csv\",\n \"/path/to/file2.csv\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": false,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"data_key\",\n \"partition_key\": \"partition_key\",\n \"timestamp_key\": \"time\",\n \"timestamp_format\": \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\": [\n \"tag1\"\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "288" + }, + { + "key": "ETag", + "value": "W/\"120-QhzA7Fga/ADDeU90Nohadt+J8fg\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:49:53 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:19:53+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"505fb3bc-ae32-4f5b-a931-adec4d1d84ba\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t41\",\n \"version_key\": \"1721220593027\"\n }\n}" } ] }, @@ -581,71 +695,88 @@ "header": [ { "key": "Accept", - "value": "application/json, text/plain, */*" + "value": "application/json, text/plain, */*", + "disabled": true }, { "key": "Accept-Language", - "value": "en-GB,en" + "value": "en-GB,en", + "disabled": true }, { "key": "Cache-Control", - "value": "no-store" + "value": "no-store", + "disabled": true }, { "key": "Connection", - "value": "keep-alive" + "value": "keep-alive", + "disabled": true }, { "key": "Content-Type", - "value": "application/json" + "value": "application/json", + "disabled": true }, { "key": "Cookie", - "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ" + "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", + "disabled": true }, { "key": "Origin", - "value": "http://localhost:3001" + "value": "http://localhost:3001", + "disabled": true }, { "key": "Pragma", - "value": "no-store" + "value": "no-store", + "disabled": true }, { "key": "Referer", - "value": "http://localhost:3001/console/dataset/new" + "value": "http://localhost:3001/console/dataset/new", + "disabled": true }, { "key": "Sec-Fetch-Dest", - "value": "empty" + "value": "empty", + "disabled": true }, { "key": "Sec-Fetch-Mode", - "value": "cors" + "value": "cors", + "disabled": true }, { "key": "Sec-Fetch-Site", - "value": "same-origin" + "value": "same-origin", + "disabled": true }, { "key": "Sec-GPC", - "value": "1" + "value": "1", + "disabled": true }, { "key": "User-Agent", - "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" + "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "disabled": true }, { "key": "sec-ch-ua", - "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"" + "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", + "disabled": true }, { "key": "sec-ch-ua-mobile", - "value": "?0" + "value": "?0", + "disabled": true }, { "key": "sec-ch-ua-platform", - "value": "\"macOS\"" + "value": "\"macOS\"", + "disabled": true } ], "body": { @@ -799,6 +930,1118 @@ "body": "{\n \"id\": \"dataset.schema.identify\",\n \"ver\": \"v1\",\n \"ts\": 1721100502574,\n \"params\": {\n \"status\": \"SUCCESS\",\n \"errmsg\": \"\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tripID' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tripID\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"VendorID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tpep_pickup_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_pickup_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_pickup_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"tpep_dropoff_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_dropoff_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_dropoff_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"passenger_count\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"trip_distance\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"RatecodeID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"store_and_fwd_flag\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"PULocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"DOLocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"payment_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primary_passenger\": {\n \"type\": \"object\",\n \"properties\": {\n \"email\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mobile\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"fare_details\": {\n \"type\": \"object\",\n \"properties\": {\n \"fare_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"extra\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mta_tax\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tip_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tolls_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"improvement_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"total_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"congestion_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 98,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" } ] + }, + { + "name": "Dataset update", + "request": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721135455988\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "response": [ + { + "name": "Success: Minimal dataset update", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "331" + }, + { + "key": "ETag", + "value": "W/\"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 13:00:45 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:30:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"354f1fec-0c39-42ee-a52a-49552f847c11\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134845559\"\n }\n}" + }, + { + "name": "Success: Updated successfully", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "331" + }, + { + "key": "ETag", + "value": "W/\"14b-y8oEEJijvIDh8wU5ogipKdkv8y0\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:57:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:27:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"6d835f07-aed5-4e8b-81c2-2142cfb55c52\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\"\n }\n}" + }, + { + "name": "Failure: Outdated key provided", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721064642580\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "383" + }, + { + "key": "ETag", + "value": "W/\"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:53:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:23:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fe03f6-c4c4-48f6-9d84-a32cd52f4c13\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_OUTDATED\",\n \"message\": \"The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates\"\n }\n}" + }, + { + "name": "Failure: Dataset not exists to update", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "340" + }, + { + "key": "ETag", + "value": "W/\"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:58:30 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:28:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf64703c-bb6b-41bf-bc1a-c85373efd925\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists with id:telemetry_record-t41\"\n }\n}" + }, + { + "name": "Failure: Invalid request", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "369" + }, + { + "key": "ETag", + "value": "W/\"171-iNJoyWUecOEsXbHZwx6rld3Sr1I\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:59:21 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:29:21+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d31672b-e5c3-4a6d-afac-d9d78011bcde\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + }, + { + "name": "Failure: No fields are provided to update", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "373" + }, + { + "key": "ETag", + "value": "W/\"175-ga30XLi7qSW4Ix+3Q/xaMUYcqII\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 13:02:44 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:32:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf99b1e1-7694-4be0-ba5d-e347764736de\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\"\n }\n}" + } + ] + }, + { + "name": "Read dataset", + "request": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "beckn-test-data" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "response": [ + { + "name": "Success: Read live dataset", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": "localhost:3007/v2/datasets/read/master-test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "501" + }, + { + "key": "ETag", + "value": "W/\"1f5-p+b/6r0nHRFhgr5+URzxk4d/CSg\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:08:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"8c8a2852-54bc-43fb-b063-7f359d11930a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n }\n}" + }, + { + "name": "Success: Read draft dataset", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "beckn-test-data" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "528" + }, + { + "key": "ETag", + "value": "W/\"210-Qo8q3dU8l7LYIXVzJwStVe90z9Q\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:11:00 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:41:00+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"96fd4f42-fa84-4730-bc79-d241a4e335a1\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n }\n}" + }, + { + "name": "Success: Read specific column", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/master-test?status=Draft&fields=name,type,id", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "master-test" + ], + "query": [ + { + "key": "status", + "value": "Draft" + }, + { + "key": "fields", + "value": "name,type,id" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "237" + }, + { + "key": "ETag", + "value": "W/\"ed-zvknH4AY6kid9Yit+KqMIdDeNGc\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:12:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:42:16+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"02a6b03a-8bf3-4e37-8dcd-859d3e8f904e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"id\": \"master-test\"\n }\n}" + }, + { + "name": "Success: Read version_key", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/beckn-test-data?fields=version_key&mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "beckn-test-data" + ], + "query": [ + { + "key": "fields", + "value": "version_key" + }, + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "211" + }, + { + "key": "ETag", + "value": "W/\"d3-hfyvp0pFPZhM2NbiiMDTfy+51YM\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:15:37 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:45:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"805e848a-d260-47c3-b55c-fc9b8323719e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1718791650227\"\n }\n}" + }, + { + "name": "Success: Read from draft, if not present cerate draft from live dataset and then read", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/sample1?mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "sample1" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "538" + }, + { + "key": "ETag", + "value": "W/\"21a-Iqg1rOMRqVpT0n8ZhQsPiTB2l44\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:19:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:49:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da70e25b-6ad0-49a7-a39d-340d1d0c46a7\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 2,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n }\n }\n}" + }, + { + "name": "Failure: Invalid field name provided", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/telemetry_record?fields=newname", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "telemetry_record" + ], + "query": [ + { + "key": "fields", + "value": "newname" + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "307" + }, + { + "key": "ETag", + "value": "W/\"133-TQ9WpmutsrDcTNkRRmbWOhUChMk\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:20:17 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:50:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\"\n }\n}" + }, + { + "name": "Failure: Dataset not found", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": "localhost:3007/v2/datasets/read/new_telemetry_record.1" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "304" + }, + { + "key": "ETag", + "value": "W/\"130-JL/oBB+GUHTrBp278giBHRvO71I\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:21:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:51:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"3aad3842-a76e-4fe8-b635-c7fef5f318f9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:new_telemetry_record.1 not found\"\n }\n}" + } + ] + }, + { + "name": "Dataset list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "response": [ + { + "name": "Success: Lists all when no filters provided", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "24340" + }, + { + "key": "ETag", + "value": "W/\"5f14-Cq3tfdk3YuXhXtjub1V0q8YVdC4\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:25:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:55:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"97efe04d-e981-493d-9ee7-a6dad6887d64\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry_record-t4\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_events\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_record-master\",\n \"name\": \"sb-telemetry\",\n \"type\": \"master\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"generate-schema\",\n \"name\": \"generate-schema\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-test\",\n \"name\": \"trip-test\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample-trip-details\",\n \"name\": \"sample-trip-details\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 30\n }\n}" + }, + { + "name": "Success: Filter based on status as array", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "5475" + }, + { + "key": "ETag", + "value": "W/\"1563-FnaUnwhv4LmMcE5J8f4+jKz+DDk\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:27:38 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" + }, + { + "name": "Success: Filter basen on status as string", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"ReadyToPublish\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "16960" + }, + { + "key": "ETag", + "value": "W/\"4240-18LMqkfsWst/M+Uc53td59PATTk\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:29:18 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:59:18+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a08c7ea0-bb1c-4998-b47d-a76e38e87e31\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 8\n }\n}" + }, + { + "name": "Success: Filter based on dataset type", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Live\",\n \"type\": \"master\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "933" + }, + { + "key": "ETag", + "value": "W/\"3a5-IM9o2EpRUzccL+l2CW/BBpoBMqA\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:30:41 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:00:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0d1ff2de-42c9-4192-b75d-84f711dbfb55\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n }\n ],\n \"count\": 2\n }\n}" + }, + { + "name": "Failure: Invalid payload", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "312" + }, + { + "key": "ETag", + "value": "W/\"138-XQplwhrgIYKIg0qtQdRCYWIGTNM\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:32:26 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:02:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"add9dbe0-f362-4f99-890c-3387c998a049\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/required must have required property 'msgid'\"\n }\n}" + } + ] } ] } diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml index a0e9c254..a29de464 100644 --- a/api-service/swagger-doc/v2_updated_doc_openapi.yml +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -111,6 +111,34 @@ paths: result: id: telemetry_record-master version_key: '1721099200603' + example-2: + summary: 'Success: Minimal request body' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-16T18:14:59+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 9e207f4f-2be6-4a45-ab78-213bea272ae0 + responseCode: OK + result: + id: telemetry_events + version_key: '1721133899306' + example-3: + summary: 'Success: Dataset created successfully with all fields' + value: + id: api.datasets.create + ver: v1 + ts: '2024-07-17T18:19:53+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 505fb3bc-ae32-4f5b-a931-adec4d1d84ba + responseCode: OK + result: + id: telemetry_record-t41 + version_key: '1721220593027' '400': description: Bad Request content: @@ -172,8 +200,8 @@ paths: /v2/files/generate-url: post: tags: - - Dataset API's - summary: API to generate presigned url's + - Dataset api's + summary: File generate url requestBody: content: application/json: @@ -289,7 +317,7 @@ paths: /dataset/v1/dataschema: post: tags: - - Dataset API's + - Dataset api's summary: Data schema generator requestBody: content: @@ -2028,4 +2056,2929 @@ paths: store_format: boolean: jsonSchema: boolean - datasource: boolean \ No newline at end of file + datasource: boolean + /v2/datasets/update: + patch: + tags: + - Dataset api's + summary: Dataset update + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.update + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + dataset_id: telemetry_record-t4 + version_key: '1721135455988' + name: sb-telemetry + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: ipid + dedup_config: + drop_duplicates: true + dedup_key: mid + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + midpid: + type: string + arrival_format: text + data_type: string + miduwi: + type: integer + arrival_format: number + data_type: epoch + mid: + type: string + arrival_format: text + data_type: string + sid: + type: string + arrival_format: text + data_type: string + additionalProperties: true + denorm_config: + denorm_fields: + - value: + denorm_key: eid + denorm_out_field: userdata + action: remove + - value: + denorm_key: eid + denorm_out_field: edata + dataset_id: trip-details + action: upsert + transformations_config: + - value: + field_key: email + transformation_function: + type: mask + expr: mid + datatype: string + category: pii + mode: Strict + action: upsert + - value: + field_key: email_id + transformation_function: + type: mask + expr: mid + datatype: string + category: pii + mode: Strict + action: remove + tags: [] + connectors_config: + - value: + connector_id: jdbc + connector_config: + source_database_type: postgresql + source_database_host: postgresql-hl.postgresql.svc.cluster.local.master + source_database_port: 5432 + source_database_name: obsrv_sample_datasets_1 + source_database_username: postgres + source_database_pwd: postgres + table: new_york_taxi_data + timestamp-column: tpep_pickup_datetime + batch-size: 100 + max-batches: 2 + operations_config: + polling_interval: periodic + schedule: twice + action: upsert + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Minimal dataset update' + value: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:30:45+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 354f1fec-0c39-42ee-a52a-49552f847c11 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134845559' + example-1: + summary: 'Success: Updated successfully' + value: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:27:55+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 6d835f07-aed5-4e8b-81c2-2142cfb55c52 + responseCode: OK + result: + message: Dataset is updated successfully + id: telemetry_record-t4 + version_key: '1721134675878' + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Invalid request' + value: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:29:21+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 7d31672b-e5c3-4a6d-afac-d9d78011bcde + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_UPDATE_INPUT_INVALID + message: >- + #properties/request/required must have required property + 'dataset_id' + example-1: + summary: 'Failure: No fields are provided to update' + value: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:32:44+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf99b1e1-7694-4be0-ba5d-e347764736de + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_UPDATE_NO_FIELDS + message: >- + Provide atleast one field in addition to the dataset_id + to update the dataset + '404': + description: Not Found + content: + application/json: + schema: + type: object + example: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:28:30+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bf64703c-bb6b-41bf-bc1a-c85373efd925 + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_EXISTS + message: Dataset does not exists with id:telemetry_record-t41 + '409': + description: Conflict + content: + application/json: + schema: + type: object + example: + id: api.datasets.update + ver: v1 + ts: '2024-07-16T18:23:16+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 02fe03f6-c4c4-48f6-9d84-a32cd52f4c13 + responseCode: CONFLICT + result: {} + error: + code: DATASET_OUTDATED + message: >- + The dataset is outdated. Please try to fetch latest changes + of the dataset and perform the updates + /v2/datasets/read/{dataset_id}: + get: + tags: + - Dataset api's + summary: Read dataset + parameters: + - name: dataset_id + in: path + required: true + schema: + type: string + example: beckn-test-data + - name: mode + in: query + schema: + type: string + example: edit + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Read live dataset' + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:38:55+05:30' + params: + status: SUCCESS + resmsgid: 8c8a2852-54bc-43fb-b063-7f359d11930a + responseCode: OK + result: + dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + example-1: + summary: 'Success: Read draft dataset' + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:41:00+05:30' + params: + status: SUCCESS + resmsgid: 96fd4f42-fa84-4730-bc79-d241a4e335a1 + responseCode: OK + result: + dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + example-2: + summary: 'Success: Read specific column' + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:42:16+05:30' + params: + status: SUCCESS + resmsgid: 02a6b03a-8bf3-4e37-8dcd-859d3e8f904e + responseCode: OK + result: + name: master-test + type: master + id: master-test + example-3: + summary: 'Success: Read version_key' + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:45:37+05:30' + params: + status: SUCCESS + resmsgid: 805e848a-d260-47c3-b55c-fc9b8323719e + responseCode: OK + result: + version_key: '1718791650227' + example-4: + summary: >- + Success: Read from draft, if not present cerate draft from + live dataset and then read + value: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:49:28+05:30' + params: + status: SUCCESS + resmsgid: da70e25b-6ad0-49a7-a39d-340d1d0c46a7 + responseCode: OK + result: + dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 2 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: false + cache_enabled: false + keys_config: + data_key: '' + timestamp_key: time + cache_config: + redis_db_host: localhost + redis_db_port: 6379 + redis_db: 0 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:50:17+05:30' + params: + status: FAILED + resmsgid: bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_INVALID_FIELDS + message: >- + The specified fields [newname] in the dataset cannot be + found. + '404': + description: Not Found + content: + application/json: + schema: + type: object + example: + id: api.datasets.read + ver: v1 + ts: '2024-07-17T17:51:12+05:30' + params: + status: FAILED + resmsgid: 3aad3842-a76e-4fe8-b635-c7fef5f318f9 + responseCode: NOT_FOUND + result: {} + error: + code: DATASET_NOT_FOUND + message: >- + Dataset with the given dataset_id:new_telemetry_record.1 not + found + /v2/datasets/list: + post: + tags: + - Dataset api's + summary: Dataset list + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.list + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: + - Live + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Lists all when no filters provided' + value: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T17:55:36+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 97efe04d-e981-493d-9ee7-a6dad6887d64 + responseCode: OK + result: + data: + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry_record-t4 + name: sb-telemetry + type: event + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_events + name: sb-telemetry + type: event + status: Draft + tags: [] + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: telemetry_record-master + name: sb-telemetry + type: master + status: Draft + tags: + - tag1 + version: 1 + api_version: v2 + dataset_config: + indexing_config: + olap_store_enabled: true + lakehouse_enabled: true + cache_enabled: false + keys_config: + data_key: '' + partition_key: '' + timestamp_key: obsrv_meta.syncts + cache_config: + redis_db_port: null + redis_db: 0 + file_upload_path: [] + - dataset_id: generate-schema + name: generate-schema + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-test + name: trip-test + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample-trip-details + name: sample-trip-details + type: event + status: Draft + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: '' + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: {} + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: {} + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 30 + example-1: + summary: 'Success: Filter based on status as array' + value: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T17:57:38+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 31aba5bc-8492-45ce-be0e-8c52d8716014 + responseCode: OK + result: + data: + - dataset_id: telemetry-summary + name: telemetry-summary + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: tripdetailstest + name: TripDetailsTest1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-normal + name: test-normal-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-dataset + name: test-dataset-renamed + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_dropoff_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: triptestdataset + name: triptestdataset + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: test-trip-details + name: test-trip-details + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + - dataset_id: sb-telemetry + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + - dataset_id: sb-telemetry-test + name: sb-telemetry + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: sb-dev.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-changes + name: test-changes + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: sample1 + name: sample1 + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: time + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-events + name: telemetry-events + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: taxt_trip + name: taxt_trip + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: date + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test + name: test + type: event + status: Live + tags: + - TAG1 + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: '' + timestamp_key: ets + exclude_fields: [] + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + count: 16 + example-2: + summary: 'Success: Filter basen on status as string' + value: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T17:59:18+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: a08c7ea0-bb1c-4998-b47d-a76e38e87e31 + responseCode: OK + result: + data: + - dataset_id: test-summary + name: test-summary + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: trip-details1 + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: tpep_pickup_datetime + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: telemetry-test-dataset + name: telemetry-test-dataset + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: null + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + exclude_fields: [] + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + - dataset_id: test-rollup + name: test-rollup + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: {} + processing: + dedupKeys: [] + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' + format type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + syncts: + type: integer + suggestions: + - message: >- + The Property 'syncts' appears to be + 'epoch' format type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + arrival_format: text + data_type: string + uid: + type: string + arrival_format: text + data_type: string + content_id: + type: string + arrival_format: text + data_type: string + context: + type: object + properties: + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + model: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + granularity: + type: string + arrival_format: text + data_type: string + date_range: + type: object + properties: + from: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.from' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.from + arrival_format: number + data_type: epoch + to: + type: integer + suggestions: + - message: >- + The Property 'context.date_range.to' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.context.properties.date_range.properties.to + arrival_format: number + data_type: epoch + arrival_format: object + data_type: object + additionalProperties: true + rollup: + type: object + arrival_format: object + data_type: object + additionalProperties: true + cdata: + type: array + items: {} + arrival_format: array + data_type: array + arrival_format: object + data_type: object + additionalProperties: true + dimensions: + type: object + properties: + did: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + sid: + type: string + arrival_format: text + data_type: string + channel: + type: string + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + mode: + type: string + arrival_format: text + data_type: string + content_type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + edata: + type: object + properties: + eks: + type: object + properties: + interact_events_per_min: + type: integer + arrival_format: number + data_type: integer + start_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.start_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.start_time + arrival_format: number + data_type: epoch + interact_events_count: + type: integer + arrival_format: number + data_type: integer + item_responses: + type: array + items: {} + arrival_format: array + data_type: array + end_time: + type: integer + suggestions: + - message: >- + The Property 'edata.eks.end_time' + appears to be 'epoch' format type. + severity: '' + path: >- + properties.edata.properties.eks.properties.end_time + arrival_format: number + data_type: epoch + events_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + page_summary: + type: array + items: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + visit_count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_diff: + type: number + arrival_format: number + data_type: number + telemetry_version: + type: string + arrival_format: text + data_type: string + env_summary: + type: array + items: + type: object + properties: + env: + type: string + arrival_format: text + data_type: string + time_spent: + type: integer + arrival_format: number + data_type: integer + count: + type: integer + arrival_format: number + data_type: integer + additionalProperties: true + arrival_format: array + data_type: array + time_spent: + type: number + arrival_format: number + data_type: number + arrival_format: object + data_type: object + additionalProperties: true + arrival_format: object + data_type: object + additionalProperties: true + tags: + type: array + items: + type: string + arrival_format: array + data_type: array + object: + type: object + properties: + ver: + type: string + arrival_format: text + data_type: string + id: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + additionalProperties: true + additionalProperties: true + - dataset_id: trip + name: trip + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: test1 + name: test1 + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + - dataset_id: beckn-test-data + name: beckn-test-data + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: ets + entry_topic: beckn-test-data + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + exclude_fields: [] + - dataset_id: trip-details + name: trip-details + type: event + status: ReadyToPublish + tags: [] + version: 1 + api_version: v2 + dataset_config: + data_key: '' + timestamp_key: obsrv_meta.syncts + entry_topic: local.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 0 + file_upload_path: [] + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + tripID: + path: $.tripID + cardinality: 99 + index: false + processing: + dedupKeys: + - tripID + dropDuplicates: + - 'Yes' + - 'No' + mergedEvent: + tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe + VendorID: '2' + tpep_pickup_datetime: '2023-04-28 00:18:42' + tpep_dropoff_datetime: '2024-02-15 00:24:38' + passenger_count: '1' + trip_distance: '1.60' + RatecodeID: '1' + store_and_fwd_flag: 'N' + PULocationID: '236' + DOLocationID: '239' + payment_type: '2' + primary_passenger: + email: Dewayne_Kuvalis17@gmail.com + mobile: 1-429-628-3797 x14211 + fare_details: + fare_amount: '7' + extra: '0.5' + mta_tax: '0.5' + tip_amount: '0' + tolls_amount: '0' + improvement_surcharge: '0.3' + total_amount: '8.3' + congestion_surcharge: '' + passenger-name: yashashk + count: 8 + example-3: + summary: 'Success: Filter based on dataset type' + value: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T18:00:41+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 0d1ff2de-42c9-4192-b75d-84f711dbfb55 + responseCode: OK + result: + data: + - dataset_id: master-test + name: master-test + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: userid + timestamp_key: '' + exclude_fields: [] + entry_topic: local.masterdata.ingest + redis_db_host: localhost + redis_db_port: 6379 + index_data: true + redis_db: 54 + - dataset_id: sb-telemetry-user + name: sb-telemetry-user + type: master + status: Live + tags: [] + version: 1 + api_version: v1 + dataset_config: + data_key: id + timestamp_key: '' + exclude_fields: [] + entry_topic: sb-dev.masterdata.ingest + redis_db_host: obsrv-redis-master.redis.svc.cluster.local + redis_db_port: 6379 + index_data: false + redis_db: 4 + count: 2 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.datasets.list + ver: v1 + ts: '2024-07-17T18:02:26+05:30' + params: + status: FAILED + resmsgid: add9dbe0-f362-4f99-890c-3387c998a049 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_LIST_INPUT_INVALID + message: >- + #properties/params/required must have required property + 'msgid' \ No newline at end of file From 98045bd8eb7e45614230482f54ac4e3ba5886015 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 17 Jul 2024 22:39:32 +0530 Subject: [PATCH 037/235] #OBS-I116: feat: Dataschema api implementation v2 --- api-service/package.json | 1 + api-service/src/v2/configs/Config.ts | 2 + .../GenerateDataSchema/GenerateDataSchema.ts | 138 +++++++++++++ .../RequestValidationSchema.json | 34 +++ .../exceptions/SchemaGenerationException.ts | 8 + api-service/src/v2/routes/Router.ts | 2 + .../SchemaGenerateService/ConfigSuggester.ts | 61 ++++++ .../SchemaGenerateService/Constants.json | 20 ++ .../DataSchemaService.ts | 106 ++++++++++ .../SchemaGenerateService/SchemaAnalyser.ts | 193 ++++++++++++++++++ .../SchemaArrayValidator.ts | 52 +++++ .../SchemaCardinalityAnalyser.ts | 50 +++++ .../SchemaGeneratorUtils.ts | 50 +++++ .../SchemaGenerateService/SchemaHandler.ts | 168 +++++++++++++++ .../SchemaGenerateService/SchemaMapping.json | 103 ++++++++++ .../SchemaGenerateService/SchemaMerger.ts | 18 ++ .../SuggestionTemplate.ts | 74 +++++++ .../SchemaGenerateService/Template.ts | 107 ++++++++++ api-service/src/v2/types/ConfigModels.ts | 16 ++ api-service/src/v2/types/IngestionModels.ts | 12 ++ api-service/src/v2/types/SchemaModel.ts | 115 +++++++++++ 21 files changed, 1330 insertions(+) create mode 100644 api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts create mode 100644 api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json create mode 100644 api-service/src/v2/exceptions/SchemaGenerationException.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/Constants.json create mode 100644 api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/SchemaMapping.json create mode 100644 api-service/src/v2/services/SchemaGenerateService/SchemaMerger.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts create mode 100644 api-service/src/v2/services/SchemaGenerateService/Template.ts create mode 100644 api-service/src/v2/types/SchemaModel.ts diff --git a/api-service/package.json b/api-service/package.json index 4359e729..fb066d1c 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -22,6 +22,7 @@ "@aws-sdk/s3-request-presigner": "^3.540.0", "@azure/storage-blob": "^12.17.0", "@google-cloud/storage": "^7.9.0", + "@jsonhero/schema-infer": "^0.1.5", "@project-sunbird/logger": "^0.0.9", "ajv": "^8.11.2", "ajv-formats": "^2.1.1", diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/v2/configs/Config.ts index 0073ae95..ce8c071d 100644 --- a/api-service/src/v2/configs/Config.ts +++ b/api-service/src/v2/configs/Config.ts @@ -56,6 +56,8 @@ export const config = { }, "exclude_datasource_validation": process.env.exclude_datasource_validation ? process.env.exclude_datasource_validation.split(",") : ["system-stats", "failed-events-summary", "masterdata-system-stats", "system-events"], // list of datasource names to skip validation while calling query API "telemetry_dataset": process.env.telemetry_dataset || `${env}.system.telemetry.events`, + "rollup_ratio": parseInt(process.env.rollup_ratio || "80"), + "unique_formats": ["uuid", "email", "uri", "ipv4", "ipv6"], "table_config": { // This object defines the configuration for each table. "datasets": { "primary_key": "id", diff --git a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts new file mode 100644 index 00000000..a47225c0 --- /dev/null +++ b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -0,0 +1,138 @@ +import httpStatus from "http-status"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { Request, Response } from "express"; +import { DataSetConfig, DatasetSchemaResponse, DatasetSchemeRequest } from "../../types/SchemaModel"; +import { obsrvError } from "../../types/ObsrvError"; +import { schemaValidation } from "../../services/ValidationService"; +import DataSchemaValidation from "./RequestValidationSchema.json" +import _ from "lodash"; +import constants from "../../services/SchemaGenerateService/Constants.json"; +import { SchemaInference } from "../../services/SchemaGenerateService/DataSchemaService"; +import { SchemaArrayValidator } from "../../services/SchemaGenerateService/SchemaArrayValidator"; +import { SchemaHandler } from "../../services/SchemaGenerateService/SchemaHandler"; +import DataMappings from "../../services/SchemaGenerateService/SchemaMapping.json"; +import { SchemaCardinalityAnalyser } from "../../services/SchemaGenerateService/SchemaCardinalityAnalyser"; +import { SuggestionTemplate } from "../../services/SchemaGenerateService/SuggestionTemplate"; +import { ConfigSuggestor } from "../../services/SchemaGenerateService/ConfigSuggester"; + +let rollupInfo = {} + +const validateRequest = async (req: Request) => { + + const isRequestValid: Record = schemaValidation(req.body, DataSchemaValidation) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATA_SCHEMA_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) + } + +} + +const dataSchema = async (req: Request, res: Response) => { + + await validateRequest(req) + const request = req.body + const dataSchemaSpec = schemaGenerate(request.data, request.config) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataSchemaSpec }); + +} + +const schemaGenerate = (sample: Map[], config: Record): any => { + const { isBatch = false, extractionKey } = config; + const isJsonSchema = checkJsonSchema(_.head(sample) || new Map()); + const schemaInference = new SchemaInference(); + const schemaArrayValidator = new SchemaArrayValidator(); + const dataset = _.get(config, "dataset") + if (isJsonSchema) { + let result = process(sample, dataset) + result.schema = removeNonIndexColumns(result.schema) + result.schema = removeFormats(result.schema) + return result + } else { + let schema = isBatch ? schemaInference.inferBatchSchema([]>sample, extractionKey) : schemaInference.inferSchema(sample); + schema = schemaArrayValidator.validate(schema) + const schemaCardinalityAnalyser = new SchemaCardinalityAnalyser(sample, schema) + rollupInfo = schemaCardinalityAnalyser.analyse() + let result = process(schema, dataset) + result.schema = removeNonIndexColumns(result.schema) + result.schema = removeFormats(result.schema) + return result + } +} + +const process = (schemas: Map[], dataset: string): DatasetSchemaResponse => { + const schemaHandler = new SchemaHandler(); + const suggestionTemplate = new SuggestionTemplate(); + const configSuggestor = new ConfigSuggestor(dataset); + const mergedSchema = schemaHandler.merge(schemas) + const report = schemaHandler.process(schemas) + const resolvedSchema = resolveConflicts(mergedSchema, report) + const suggestionTemplates = suggestionTemplate.createSuggestionTemplate(report) + const schema = schemaHandler.update(resolvedSchema, suggestionTemplates, "suggestions") + const suggestedConfig: DataSetConfig = configSuggestor.suggestConfig(report, rollupInfo) + const updatedSchema = schemaHandler.mapDataTypes(schema) + _.set(updatedSchema, "additionalProperties", true); + return { "schema": updatedSchema, "configurations": suggestedConfig, "dataMappings": DataMappings } +} + + +const checkJsonSchema = (sample: Map): boolean => { + const schemaProps = ["$id", "$schema", "$ref"] + return Object.keys(sample).some(key => schemaProps.includes(key)); +} + +const removeNonIndexColumns = (schema: any) => { + if (schema.properties) { + Object.entries(schema.properties).map(([key, property]: any) => { + _.unset(schema, 'required'); + removeNonIndexColumns(property) + }); + } else if (schema.items) { + removeNonIndexColumns(schema.items) + } + if (Array.isArray(schema.required) && schema.required.length === 0) { + _.unset(schema, 'required'); + } + return schema +} + +const removeFormats = (schema: any) => { + if (schema.properties) { + Object.entries(schema.properties).map(([key, property]: any) => { + // Removing format to avoid schema validation issues + const isDateTypeField = ['date-time', 'date', 'epoch'].includes((property as any).format); + if (isDateTypeField && _.get(property, 'data_type') === 'string') { + _.set(property, 'data_type', _.get(property, 'format')); + } else if (isDateTypeField && _.get(property, 'data_type') === 'integer') { + _.set(property, 'data_type', 'epoch'); + } + _.unset(property, 'format'); + removeFormats(property) + }); + } else if (schema.items) { + _.unset(schema.items, 'format'); + removeFormats(schema.items) + } + return schema +} + +const resolveConflicts = (schema: any, updateProps: any): any => { + const schemaHandler = new SchemaHandler(); + updateProps.map((value: any) => { + if (!_.isEmpty(value.schema) || !_.isEmpty(value.required)) { + switch (value.schema.type || value.required.type) { + case constants.SCHEMA_RESOLUTION_TYPE.DATA_TYPE: + return schemaHandler.update(schema, value, "datatype") + case constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD: + return schemaHandler.update(schema, value, "setNulltype") + case constants.SCHEMA_RESOLUTION_TYPE.OPTIONAL_TYPE: + return schemaHandler.update(schema, value, "required") + default: + console.warn("Unsupported Conflict Type") + break; + } + } else { console.info(`Conflicts not found ${JSON.stringify(value)}`) } + }) + return schema +} + + +export default dataSchema; \ No newline at end of file diff --git a/api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json b/api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json new file mode 100644 index 00000000..0a882400 --- /dev/null +++ b/api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json @@ -0,0 +1,34 @@ +{ + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "object" + }, + "minItems": 1 + }, + "config": { + "type": "object", + "properties": { + "dataset": { + "type": "string" + }, + "isBatch": { + "type": "boolean" + }, + "extractionKey": { + "type": "string" + } + }, + "required": [ + "dataset" + ] + } + }, + "required": [ + "data", + "config" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/api-service/src/v2/exceptions/SchemaGenerationException.ts b/api-service/src/v2/exceptions/SchemaGenerationException.ts new file mode 100644 index 00000000..44805da4 --- /dev/null +++ b/api-service/src/v2/exceptions/SchemaGenerationException.ts @@ -0,0 +1,8 @@ +export class SchemaGenerationException extends Error { + statusCode: number; + constructor(message: string, code: number) { + super(message); + this.name = 'SchemaGenerationException'; + this.statusCode = code; + } +} \ No newline at end of file diff --git a/api-service/src/v2/routes/Router.ts b/api-service/src/v2/routes/Router.ts index ebdbe423..c1df4aed 100644 --- a/api-service/src/v2/routes/Router.ts +++ b/api-service/src/v2/routes/Router.ts @@ -20,6 +20,7 @@ import { eventValidation } from "../controllers/EventValidation/EventValidation" import GenerateSignedURL from "../controllers/GenerateSignedURL/GenerateSignedURL"; import { sqlQuery } from "../controllers/QueryWrapper/SqlQueryWrapper"; import DatasetStatusTansition from "../controllers/DatasetStatusTransition/DatasetStatusTransition"; +import DataSchemaGenerator from "../controllers/GenerateDataSchema/GenerateDataSchema"; export const router = express.Router(); @@ -39,6 +40,7 @@ router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); diff --git a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts new file mode 100644 index 00000000..14a006d4 --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts @@ -0,0 +1,61 @@ +import _ from "lodash"; +import { DataSetConfig, DatasetProcessing } from "../../types/ConfigModels"; +import { IngestionConfig, RollupInfo } from "../../types/IngestionModels"; +import { ConflictTypes } from "../../types/SchemaModel"; +import { ingestionConfig } from "../../configs/IngestionConfig"; +import { parseSchemaPath } from "./SchemaGeneratorUtils"; + + +export const defaultLabels = { + dropDuplicates: ["Yes", "No"] +} + +export class ConfigSuggestor { + /** + * Responsiblities : + * 1. Suggest rollup is required or not. - done + * 2. Suggest the dedup property fields. - done + */ + private dataset: string + private rollupInfo: RollupInfo + constructor(dataset: string) { + this.dataset = dataset + this.rollupInfo = {} + } + + public suggestConfig(conflicts: ConflictTypes[], rollupInfo: RollupInfo): DataSetConfig { + this.rollupInfo = rollupInfo + const suggestedConfig = this.analyzeConflicts(conflicts) + return suggestedConfig + } + + private analyzeConflicts(conflicts: ConflictTypes[]): DataSetConfig { + const typeFormatsConflict: ConflictTypes[] = _.filter(conflicts, (o) => !_.isEmpty(o.formats)); + const ingestionConfig: IngestionConfig = this.ingestionConfig(typeFormatsConflict) + const processingConfig: DatasetProcessing = this.processingConfig(typeFormatsConflict) + return { "indexConfiguration": ingestionConfig, "processing": processingConfig } + } + + private ingestionConfig(conflicts: ConflictTypes[]): any { + return { "index": Object.assign(ingestionConfig.indexCol), "rollupSuggestions": this.rollupInfo }; + } + + private processingConfig(conflicts: ConflictTypes[]): any { + let dedupKeys = _.filter(conflicts, (o) => _.upperCase(o.formats.resolution["type"]) === "DEDUP").map(v => v.formats.property) + let matchedDedupFields = [] + let dedupOrderProperty: string = "cardinality" + let dedupOrder: any = "desc" + if (!_.isUndefined(this.rollupInfo.summary)) { + for (const key of Object.keys(this.rollupInfo.summary)) { + if (!this.rollupInfo.summary[key].index) { + for (const dedupKey of dedupKeys) { + if (dedupKey == parseSchemaPath(this.rollupInfo.summary[key].path)) matchedDedupFields.push(this.rollupInfo.summary[key]) + } + } + } + matchedDedupFields = _.orderBy(matchedDedupFields, dedupOrderProperty, dedupOrder) + dedupKeys = _.map(matchedDedupFields, matchedDedupField => parseSchemaPath(matchedDedupField.path)); + } + return { "dedupKeys": dedupKeys, dropDuplicates: defaultLabels.dropDuplicates } + } +} diff --git a/api-service/src/v2/services/SchemaGenerateService/Constants.json b/api-service/src/v2/services/SchemaGenerateService/Constants.json new file mode 100644 index 00000000..ba64c940 --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/Constants.json @@ -0,0 +1,20 @@ +{ + "SCHEMA_RESOLUTION_TYPE": { + "OPTIONAL_TYPE": "OPTIONAL_TYPE", + "FORMAT_TYPE": "FORMAT_TYPE", + "DATA_TYPE": "DATA_TYPE", + "INDEX": "INDEX", + "TRANSFORM": "TRANSFORM", + "DROP": "DROP", + "DEDUP": "DEDUP", + "NULL_FIELD": "NULL_FIELD", + "ARRIVAL_FORMAT": "ARRIVAL_FORMAT" + }, + "SEVERITY": { + "CRITICAL": "CRITICAL", + "MEDIUM": "MEDIUM", + "HIGH": "HIGH", + "LOW": "LOW", + "MUST-FIX": "MUST-FIX" + } +} diff --git a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts new file mode 100644 index 00000000..8a77344d --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts @@ -0,0 +1,106 @@ +import { inferSchema } from "@jsonhero/schema-infer"; +import httpStatus from "http-status"; +import _ from "lodash"; +import moment from "moment"; +import { SchemaGenerationException } from "../../exceptions/SchemaGenerationException"; + +const DATE_FORMATS = [ + 'MM/DD/YYYY','DD/MM/YYYY', 'YYYY-MM-DD', 'YYYY-DD-MM', 'YYYY/MM/DD', + 'DD-MM-YYYY', 'MM-DD-YYYY', 'MM-DD-YYYY HH:mm:ss', 'YYYY/MM/DD HH:mm:ss', + 'YYYY-MM-DD HH:mm:ss', 'YYYY-DD-MM HH:mm:ss', 'DD/MM/YYYY HH:mm:ss', + 'DD-MM-YYYY HH:mm:ss', 'MM-DD-YYYY HH:mm:ss.SSS', 'YYYY-MM-DD HH:mm:ss.SSS', + 'YYYY-DD-MM HH:mm:ss.SSS', 'YYYY/MM/DD HH:mm:ss.SSS', 'DD/MM/YYYY HH:mm:ss.SSS', + 'DD-MM-YYYY HH:mm:ss.SSS', 'DD-MM-YYYYTHH:mm:ss.SSSZ', 'YYYY-MM-DDTHH:mm:ss.SSSZ', + 'YYYY-DD-MMTHH:mm:ss.SSSZ', 'YYYY/MM/DDTHH:mm:ss.SSSZ', 'DD/MM/YYYYTHH:mm:ss.SSSZ', + 'YYYY-DD-MMTHH:mm:ss.SSS', 'YYYY/MM/DDTHH:mm:ss.SSS', 'DD/MM/YYYYTHH:mm:ss.SSS', + 'MM-DD-YYYYTHH:mm:ss.SSSZ', 'DD-MM-YYYYTHH:mm:ssZ', 'YYYY-MM-DDTHH:mm:ssZ', + 'YYYY-DD-MMTHH:mm:ssZ', 'YYYY/MM/DDTHH:mm:ssZ', 'DD/MM/YYYYTHH:mm:ssZ', 'MM-DD-YYYYTHH:mm:ssZ', + 'MM-DD-YYYYTHH:mm:ss', 'DD-MM-YYYYTHH:mm:ss', 'YYYY-MM-DDTHH:mm:ss', 'YYYY-DD-MMTHH:mm:ss', + 'YYYY/MM/DDTHH:mm:ss', 'DD/MM/YYYYTHH:mm:ss', 'DD-MM-YYYY HH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss.SSSZ', + 'YYYY-DD-MM HH:mm:ss.SSSZ', 'YYYY/MM/DD HH:mm:ss.SSSZ', 'DD/MM/YYYY HH:mm:ss.SSSZ', + 'MM-DD-YYYY HH:mm:ss.SSSZ', 'DD-MM-YYYY HH:mm:ssZ', 'YYYY-MM-DD HH:mm:ssZ', 'YYYY-DD-MM HH:mm:ssZ', + 'YYYY/MM/DD HH:mm:ssZ', 'DD/MM/YYYY HH:mm:ssZ', 'MM-DD-YYYY HH:mm:ssZ', 'DD-MM-YYYYTHH:mm:ss.SSSSSSZ', + 'YYYY-MM-DDTHH:mm:ss.SSSSSSZ', 'YYYY-DD-MMTHH:mm:ss.SSSSSSZ', 'YYYY/MM/DDTHH:mm:ss.SSSSSSZ', + 'DD/MM/YYYYTHH:mm:ss.SSSSSSZ', 'MM-DD-YYYYTHH:mm:ss.SSSSSSZ', 'DD/MM/YYYYTHH:mm:ss.SSSSSS', + 'YYYY-DD-MMTHH:mm:ss.SSSSSS', 'YYYY/MM/DDTHH:mm:ss.SSSSSS', 'YYYY-MM-DDTHH:mm:ss.SSSSSS', + 'MM-DD-YYYYTHH:mm:ss.SSSSSS', 'DD-MM-YYYYTHH:mm:ss.SSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSS', + 'YYYY-MM-DD HH:mm:ss.SSSSSS', 'YYYY-DD-MM HH:mm:ss.SSSSSS', 'YYYY/MM/DD HH:mm:ss.SSSSSS', + 'DD/MM/YYYY HH:mm:ss.SSSSSS', 'MM-DD-YYYY HH:mm:ss.SSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSSZ', + 'YYYY-MM-DDTHH:mm:ss.SSSSSSSSSZ', 'YYYY-DD-MMTHH:mm:ss.SSSSSSSSSZ', 'YYYY/MM/DDTHH:mm:ss.SSSSSSSSSZ', + 'DD/MM/YYYYTHH:mm:ss.SSSSSSSSSZ', 'MM-DD-YYYYTHH:mm:ss.SSSSSSSSSZ', 'DD/MM/YYYYTHH:mm:ss.SSSSSSSSS', + 'YYYY-DD-MMTHH:mm:ss.SSSSSSSSS', 'YYYY/MM/DDTHH:mm:ss.SSSSSSSSS', 'YYYY-MM-DDTHH:mm:ss.SSSSSSSSS', + 'MM-DD-YYYYTHH:mm:ss.SSSSSSSSS', 'DD-MM-YYYYTHH:mm:ss.SSSSSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSSSSS', + 'YYYY-MM-DD HH:mm:ss.SSSSSSSSS', 'YYYY-DD-MM HH:mm:ss.SSSSSSSSS', 'YYYY/MM/DD HH:mm:ss.SSSSSSSSS', + 'DD/MM/YYYY HH:mm:ss.SSSSSSSSS', 'MM-DD-YYYY HH:mm:ss.SSSSSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSSSSSZ', + 'DD-MM-YYYYTHH:mm:ss.SSSSSSSSSZ', +]; + +export class SchemaInference { + + public inferSchema(sample: any) { + const schema = _.map(sample, (value): any => this.validateEpoch(inferSchema(value).toJSONSchema({ includeSchema: true }), value, "properties")) + return schema + } + + public inferBatchSchema(sample: Map[], extractionKey: string) { + return _.flatMap(sample, (value) => { + if (extractionKey) { + const extracted = _.get(value, extractionKey); + if (extracted) { + return this.inferSchema(extracted); + } else { + throw new SchemaGenerationException('Unable to extract the batch data.', httpStatus.BAD_REQUEST); + } + } else { + throw new SchemaGenerationException('Extraction key not found.', httpStatus.BAD_REQUEST); + } + }) + } + + private validateEpoch(schema: any, sample: any, path: any) { + Object.entries(sample).map(([key, value]) => { + if (value && typeof value == 'object') { + this.validateEpoch(schema, value, `${path}.${key}.properties`) + } + const { isValidTimestamp, type } = this.isValidTimestamp(value); + const format = _.get(schema, `${path}.${key}.format`); + if (isValidTimestamp) { + _.set(schema, `${path}.${key}.format`, type) + } + else if(format && ["date-time", "time", "date"].includes(format) && !isValidTimestamp) { + _.unset(schema, `${path}.${key}.format`) + } + }); + return schema + } + + isValidTimestamp(value: any) { + const dataType = typeof value; + switch (dataType) { + case 'string': + const epochRegex = /^\d+$/ig; + if(epochRegex.test(value)){ + const parsedValue = parseInt(value, 10); + // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds + return { + isValidTimestamp: parsedValue >= 946684800 && moment(parsedValue).isValid(), + type: "epoch" + } + } else return { + isValidTimestamp: moment(value, DATE_FORMATS, true).isValid(), + type: "date-time" + } + case 'number': + // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds + return { + isValidTimestamp: value >= 946684800 && moment(value).isValid(), + type: "epoch" + }; + default: + return { + isValidTimestamp: false, + type: "" + }; + } + } +} \ No newline at end of file diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts new file mode 100644 index 00000000..14ee0e3d --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts @@ -0,0 +1,193 @@ +import _ from "lodash"; +import { Conflict, ConflictTypes, FlattenSchema, Occurance } from "../../types/SchemaModel"; +import constants from "./Constants.json"; +export class SchemaAnalyser { + private transformationCols = ["email", "creditcard", "ipv4", "ipv6"] + private dateFormatCols = ["date", "date-time"] + private cardinalCols = ["uuid"] + + private schemas: Map[]; + private minimumSchemas: number = 1 + + + constructor(schemas: Map[]) { + this.schemas = schemas; + } + + /** + * Method to analyse the schema + */ + public analyseSchema(): ConflictTypes[] { + return this.findConflicts() + } + + private findConflicts(): ConflictTypes[] { + const result: FlattenSchema[] = _.flatten(this.schemas.map(element => { + return this.flattenSchema(new Map(Object.entries(element))); + })) + const conflicts = Object.entries(_.groupBy(result, 'path')).map(([key, value]) => { + return this.getSchemaConflictTypes(this.getOccurance(value, key)) + }) + return _.filter(conflicts, obj => (!_.isEmpty(obj.schema) || !_.isEmpty(obj.required) || !_.isEmpty(obj.formats))) + } + + /** + * Retruns possible conflicts types + * 1. data type conflicts + * 2. Data format conflicts + * 3. Optional property conflicts + */ + private getSchemaConflictTypes(occuranceObj: Occurance): ConflictTypes { + const absolutePath = _.head(_.keysIn(occuranceObj.absolutePath)) + const updatedPath = absolutePath ? _.replace(absolutePath, "$.", "") : ""; + let schemaConflicts = this.findDataTypeConflicts(occuranceObj,) + const requiredConflicts = (_.size(this.schemas) > this.minimumSchemas) ? this.findOptionalPropConflicts(occuranceObj) : {} + const formatConflict = this.findFormatConflicts(occuranceObj) + if(_.size(_.keys(schemaConflicts)) > 0) { + schemaConflicts = { ...schemaConflicts, path: updatedPath } + } + return { "schema": schemaConflicts, "required": requiredConflicts, "formats": formatConflict, "absolutePath": updatedPath } + } + + /** + * Method to get the data type conflicts + */ + private findDataTypeConflicts(occurance: Occurance): Conflict { + if(_.includes(_.keys(occurance.dataType), "null") && _.size(occurance.dataType) === 1) { + return { + type: constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD, + // Should be used only to return the name of field instead of path + // property: Object.keys(occurance.property)[0], + property: _.replace(Object.keys(occurance.path)[0], "$.", ""), + conflicts: occurance.dataType, + resolution: { "value": occurance.dataType, "type": constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD }, + values: _.keys(occurance.dataType), + severity: constants.SEVERITY["MUST-FIX"], + path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), + } + } + const minimumOccurance: number = 1 + if (_.size(occurance.dataType) > minimumOccurance) { + const isUnresolvable: boolean = _.uniq(_.values(occurance.dataType)).length === 1; + const highestValueKey = !isUnresolvable ? Object.keys(occurance.dataType).reduce((a, b) => occurance.dataType[a] > occurance.dataType[b] ? a : b) : undefined + return { + type: constants.SCHEMA_RESOLUTION_TYPE.DATA_TYPE, + property: _.replace(Object.keys(occurance.path)[0], "$.", ""), + conflicts: occurance.dataType, + resolution: { "value": highestValueKey, "type": constants.SCHEMA_RESOLUTION_TYPE.DATA_TYPE }, + values: _.without(_.keys(occurance.dataType), "null"), + severity: constants.SEVERITY["MUST-FIX"], + path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), + } + } else { return {} } + } + + /** + * Method to get the format type conflicts + * + */ + private findFormatConflicts(occurance: Occurance): Conflict { + const filteredFormat = _.omit(occurance.format, "undefined") + const formats = _.concat(this.transformationCols, this.dateFormatCols, this.cardinalCols, ["uri"]); + if (!_.isEmpty(filteredFormat)) { + const formatName = _.filter(formats, f => _.has(filteredFormat, f)); + const suggestedFormat = this.idenfityFormat(formatName[0]) + return { + type: formatName[0], + property: _.replace(Object.keys(occurance.path)[0], "$.", ""), + conflicts: filteredFormat, + resolution: { "value": formatName, "type": suggestedFormat.type }, + values: _.keys(filteredFormat), + severity: suggestedFormat.severity || "", + path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), + } + } else { return {} } + + } + + + private idenfityFormat(value: string) { + if (_.includes(this.transformationCols, value)) { + return { type: "TRANSFORMATION", "severity": "LOW" }; + } else if (_.includes(this.dateFormatCols, value)) { + return { type: "INDEX", "severity": "LOW" }; + } else if (_.includes(this.cardinalCols, value)) { + return { type: "DEDUP", "severity": "LOW" }; + } else { + return {}; + } + } + + /** + * + */ + private findOptionalPropConflicts(occurance: Occurance): Conflict { + const maxOccurance: number = 1 + const requiredCount = _.map(occurance.property, (value, key) => { + return value + })[0] + + const highestValueKey = Boolean(Object.keys(occurance.isRequired).reduce((a, b) => occurance.isRequired[a] > occurance.isRequired[b] ? a : b)) + const isPropertyRequired = requiredCount <= maxOccurance ? false : true + if (highestValueKey != isPropertyRequired) { + return { + type: constants.SCHEMA_RESOLUTION_TYPE.OPTIONAL_TYPE, + property: Object.keys(occurance.property)[0], + conflicts: occurance.property, + resolution: { "value": (isPropertyRequired), "type": "OPTIONAL" }, + values: [true, false], + severity: "MEDIUM", + path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), + } + } + else { return {} } + + } + + /** + * + * Method to get the occurance of the given key from the given object + */ + private getOccurance(arrayOfObjects: object[], key: string): Occurance { + const result = _(arrayOfObjects).flatMap(obj => _.toPairs(obj)).groupBy(([key, value]) => key) + .mapValues(group => _.countBy(group, ([key, value]) => value)).value(); + return { property: result.property, dataType: result.dataType, isRequired: result.isRequired, path: result.path, absolutePath: result.absolutePath, format: result.formate }; + } + + /** + * Method to iterate over the schema object in a recursive and flatten the required properties + */ + public flattenSchema(sample: Map): FlattenSchema[] { + let array = new Array(); + const recursive = (data: any, path: string, requiredProps: string[], schemaPath: string) => { + _.map(data, (value, key) => { + let isMultipleTypes = ''; + if(_.has(value, 'anyOf')) isMultipleTypes = 'anyOf'; + if(_.has(value, 'oneOf')) isMultipleTypes = 'oneOf'; + if (_.isPlainObject(value) && (_.has(value, 'properties'))) { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) + recursive(value['properties'], `${path}.${key}`, value['required'], `${schemaPath}.properties.${key}`); + } else if(_.isPlainObject(value)) { + if (value.type === 'array') { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) + if (_.has(value, 'items') && _.has(value["items"], 'properties')) { + recursive(value["items"]['properties'], `${path}.${key}[*]`, value["items"]['required'], `${schemaPath}.properties.${key}.items`); + } + } else if(isMultipleTypes != '') { + array.push(this._flattenSchema(key, value[isMultipleTypes][0].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) + array.push(this._flattenSchema(key, value[isMultipleTypes][1].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) + } else { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) + } + } + }) + } + recursive(sample.get("properties"), "$", sample.get("required"), "$") + return array + } + + private _flattenSchema(expr: string, objType: string, isRequired: boolean, path: string, schemaPath: string, formate: string): FlattenSchema { + return { "property": expr, "dataType": objType, "isRequired": isRequired, "path": path, "absolutePath": schemaPath, "formate": formate } + } + +} diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts new file mode 100644 index 00000000..cc52309e --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts @@ -0,0 +1,52 @@ +import _, { isUndefined } from "lodash"; + +export class SchemaArrayValidator { + public validate(schemas: any) { + _.map(schemas, (schema: any, index: number) => { + Object.entries(schema).map(([schemaKey, schemaValue]) => { + if (typeof schemaValue === 'object') { + this.handleNestedObject(index, `${schemaKey}`, schemaValue, schemas); + } + }); + }); + return schemas + } + + private checkForInvalidArray(value: any) { + if (_.has(value, 'items') && _.has(value, 'properties')) + _.unset(value, 'properties'); + } + + private handleNestedObject(index: any, path: string, value: any, schemas: any) { + Object.entries(value).map(([nestedKey, nestedValue]: any) => { + if (typeof nestedValue === 'object') { + this.handleNestedObject(index, `${path}.${nestedKey}`, nestedValue, schemas) + } else if (nestedValue.type === 'array' && (nestedValue.items != false)) { + this.checkForInvalidArray(nestedValue); + let isValidArray = true; + if(_.isEqual(_.get(schemas[0], `${path}.${nestedKey}.type`), _.get(schemas[index], `${path}.${nestedKey}.type`))) { + isValidArray = _.isEqual( + _.get(schemas[0], `${path}.${nestedKey}.items`), + _.get(schemas[index], `${path}.${nestedKey}.items`) + ) + } + if (!isValidArray) { + this.deleteItemsAndSetAdditionalProperties(schemas, `${path}.${nestedKey}`) + } + } else if (nestedValue.type === 'array' && (nestedValue.items == false)) { + this.deleteItemsAndSetAdditionalProperties(schemas, `${path}.${nestedKey}`) + } + }) + } + + private deleteItemsAndSetAdditionalProperties(schemas: any, path: string) { + _.map((schemas), (schema: any, index: number) => { + if (!isUndefined(_.get(schema, path))) { + _.unset(schema, `${path}`) + _.set(schema, `${path}.type`, "array"); + _.set(schema, `${path}.additionalProperties`, false); + } + }); + } + +} diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts new file mode 100644 index 00000000..23dee0cb --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts @@ -0,0 +1,50 @@ +import { SchemaMerger } from "./SchemaMerger"; +import _ from "lodash"; +import { generateRollupSummary, generateSchemaPath, updateCardinalColumns } from "./SchemaGeneratorUtils"; +import { FieldSchema, UniqueValues } from "../../types/SchemaModel"; +export class SchemaCardinalityAnalyser { + private sampleData: Record[] + private sampleSchema: Map[] + private schemaMerger: SchemaMerger + private mergedSchema: FieldSchema + private uniqueValues: UniqueValues + constructor(sampleData: Record[], sampleSchema: Map[]) { + this.sampleData = sampleData + this.sampleSchema = sampleSchema + this.schemaMerger = new SchemaMerger() + this.mergedSchema = this.schemaMerger.mergeSchema(this.sampleSchema) || {} + this.uniqueValues = {} + } + + public analyse() { + this.generateUniqueValues(this.sampleData, this.mergedSchema) + return generateRollupSummary(this.uniqueValues) + } + + private generateUniqueValues(data: any, schema: FieldSchema, currentPath: string = "") { + if (!_.isUndefined(data) && !_.isUndefined(schema) && _.isArray(data)) { + for (const item of data) { + if (schema?.type === "object" && !_.isUndefined(schema.properties)) { + this.identifyCardinalColumns(item, schema.properties, currentPath); + } else if (schema?.type === "array" && !_.isUndefined(schema.items)) { + this.generateUniqueValues(item, schema.items, currentPath); + } + } + } + } + + private identifyCardinalColumns(data: Record, schema: any, currentPath: string) { + if (!_.isUndefined(data) && !_.isUndefined(schema)) { + for (const key of Object.keys(schema)) { + const fieldSchema = schema[key] + const path = generateSchemaPath(currentPath, key) + if (fieldSchema?.type === "object") { + this.identifyCardinalColumns(data[key], fieldSchema.properties, path); + } else if (fieldSchema?.type === "array") { + this.generateUniqueValues(data[key], fieldSchema.items || {}, `${path}[*]`); + } + updateCardinalColumns(data, fieldSchema, currentPath, key, this.uniqueValues) + } + } + } +} diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts new file mode 100644 index 00000000..832ce7a5 --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts @@ -0,0 +1,50 @@ +import _ from "lodash"; +import { config } from "../../configs/Config"; +import { UniqueValues, FieldSchema, RollupSummary } from "../../types/SchemaModel"; + + +export const generateRollupSummary = (uniqueValues: UniqueValues) => { + const summary: RollupSummary = {}; + Object.entries(uniqueValues).map(([field, value]) => { + let data: Record = {}; + _.map(value, (item: string) => { + if (!_.has(data, [field, item])) _.set(data, [field, item], 1); + else data[field][item] += 1; + }); + const resultData: Record = {}; + _.map(_.keys(data), (path: string) => { + Object.entries(data[path]).map(([key, value]: any) => { + const totalValue = _.sum(_.values(data[path])); + const ratio = Math.round((value / totalValue) * 100); + if (!_.has(resultData, path)) _.set(resultData, path, ratio); + else if (ratio > _.get(resultData, path)) + _.set(resultData, path, ratio); + }); + let fieldName = parseSchemaPath(path); + summary[fieldName] = { + path: `$.${path}`, + cardinality: 100 - _.get(resultData, path), + index: _.get(resultData, path) >= config.rollup_ratio, + }; + }); + }); + return { summary }; +}; + +export const updateCardinalColumns = (data: Record, fieldSchema: FieldSchema, path: string, key: string, uniqueValues: UniqueValues) => { + const schemaPath = generateSchemaPath(path, key); + if (_.includes(config.unique_formats, fieldSchema.format)) { + const value = _.get(data, key); + if (!_.has(uniqueValues, [schemaPath])) uniqueValues[schemaPath] = [value]; + else uniqueValues[schemaPath].push(value); + } + return uniqueValues; +}; + +export const generateSchemaPath = (path: string, key: string) => { + return (path ? `${path}.properties.${key}` : key).replace(/\[\*\]/gi, ".items"); +}; + +export const parseSchemaPath = (path: string) => { + return path.replace("$.", "").replace(/.properties/gi, "").replace(/.items/gi, "[*]"); +}; diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts new file mode 100644 index 00000000..ad3bb713 --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts @@ -0,0 +1,168 @@ +import _ from "lodash"; +import { ConflictTypes, SuggestionsTemplate } from "../../types/SchemaModel"; +import { SchemaMerger } from "./SchemaMerger"; +import { SchemaAnalyser } from "./SchemaAnalyser"; +import DataMappings from "./SchemaMapping.json"; + +export const dataMappingPaths = { + "string": "text.store_format.string.jsonSchema", + "date-time": "text.store_format.date-time.jsonSchema", + "date": "text.store_format.date.jsonSchema", + "boolean": "boolean.store_format.boolean.jsonSchema", + "integer": "number.store_format.integer.jsonSchema", + "number": "number.store_format.number.jsonSchema", + "object": "object.store_format.object.jsonSchema", + "array": "array.store_format.array.jsonSchema", +} + +export class SchemaHandler { + private typeToMethod = { + required: this.updateRequiredProp.bind(this), + datatype: this.updateDataTypes.bind(this), + suggestions: this.setSuggestions.bind(this), + setNulltype: this.setNulltype.bind(this) + } + + public process(schema: Map[]): ConflictTypes[] { + const schemaAnalyser = new SchemaAnalyser(schema) + return schemaAnalyser.analyseSchema() + } + + public merge(schema: any): any { + const schemaMerger = new SchemaMerger() + return schemaMerger.mergeSchema(schema) + } + + public update(schema: any, property: any, type: keyof typeof this.typeToMethod) { + const method = this.typeToMethod[type]; + return method(schema, property); + } + + private updateDataTypes(schema: any, conflict: ConflictTypes): any { + const { absolutePath, schema: { resolution: { value } } } = conflict; + return _.set(schema, `${absolutePath}`, { + ...schema[absolutePath], + ...{ + type: conflict.schema.resolution["value"], + oneof: conflict.schema.values.map(key => ({ type: key })), + } + }); + } + + private setNulltype(schema: any, conflict: ConflictTypes): any { + const { absolutePath, schema: { resolution: { value } } } = conflict; + const dataTypes: any = []; + _.forEach(DataMappings, (valueItem, keyItem) => { + _.forEach(_.get(valueItem, 'store_format'), (subValue, subKey) => { + if (!_.find(dataTypes, ["type", subValue["jsonSchema"]])) + dataTypes.push({ type: subValue["jsonSchema"] }) + }) + }); + const arrivalDataTypes: any = _.keys(DataMappings).map((key: any) => ({ type: key })); + + _.set(schema, `${absolutePath}.type`, 'null'); + _.set(schema, `${absolutePath}.arrivalOneOf`, arrivalDataTypes); + return _.set(schema, `${absolutePath}.oneof`, dataTypes); + } + + private updateRequiredProp(schema: any, value: ConflictTypes): any { + const absolutePath = value.absolutePath.replace(value.required.property, value.required.property.replace('.', '$')) + const subStringArray: string[] = _.split(absolutePath, '.'); + const subString: string = _.join(_.slice(subStringArray, 0, subStringArray.length - 2), '.'); + const path: string = _.isEmpty(subString) ? "required" : `${subString}.required` + const requiredList: string[] = _.get(schema, path) + const newProperty: string = value.required.property + value.required.resolution.value ? _.concat(_.get(schema, path), value.required.property) : _.pull(requiredList, newProperty) + return _.set(schema, path, _.uniq(requiredList)) + } + + private getArrivalSuggestions(schema: any, fieldData: any, property: any, type: string) { + let arrivalSuggestions: any = []; + const types = _.get(fieldData, type); + types && types.map((item: any) => { + const storeFormat = _.get(dataMappingPaths, item.type); + arrivalSuggestions.push({ type: _.first(storeFormat.split(".")) }); + }) + if (arrivalSuggestions.length > 0) + _.set(schema, `${property}.arrivalOneOf`, arrivalSuggestions); + return; + }; + + private getArrivalFormat(schema: any, fieldData: any, property: any, type: string) { + const types = _.get(fieldData, type); + types && types.map((item: any) => { + const storeFormat = _.get(dataMappingPaths, item.type); + _.set(schema, `${property}.arrival_format`, _.first(storeFormat.split("."))); + _.set(schema, `${property}.data_type`, _.get(DataMappings, storeFormat)); + }) + return; + } + + private setSuggestions(schema: any, suggestedTemplate: SuggestionsTemplate[]): any { + suggestedTemplate && suggestedTemplate.map(({ property, suggestions }) => { + const fieldData = _.get(schema, property); + _.set(schema, `${property}.suggestions`, suggestions); + const arrivalConflictExists = _.filter(suggestions, (suggestion) => _.has(suggestion, "arrivalConflict")); + switch (true) { + // Add arrival conflicts if there is arrival conflict in suggestions + case _.has(fieldData, 'oneof') && arrivalConflictExists.length > 0: + return this.getArrivalSuggestions(schema, fieldData, property, 'oneof') + // Add arrival type if there are no arrival type conflicts + case arrivalConflictExists.length === 0: + return this.getArrivalFormat(schema, fieldData, property, 'oneof') + default: + break; + } + }); + return schema; + } + + private updateStoreType(schemaValue: any, dataType: string): any { + const storeFormat = _.get(dataMappingPaths, dataType); + if (storeFormat) { + _.set(schemaValue, "arrival_format", _.first(storeFormat.split("."))); + if (!_.get(schemaValue, "data_type")) _.set(schemaValue, "data_type", _.get(DataMappings, storeFormat)); + } + } + + private checkForInvalidArray(value: any) { + if (_.has(value, 'items') && _.has(value, 'properties')) + _.unset(value, 'properties'); + } + + private updateMappings(schema: Map) { + const recursive = (data: any) => { + _.map(data, (value, key) => { + if (_.isPlainObject(value)) { + if ((_.has(value, 'properties'))) { + recursive(value['properties']); + } + if (value.type === 'array') { + if (_.has(value, 'items') && _.has(value["items"], 'properties')) { + recursive(value["items"]['properties']); + } + if (_.has(value, 'items') && _.has(value, 'properties')) + this.checkForInvalidArray(value); + this.updateStoreType(value, _.get(value, "type")); + } else { + if (_.get(value, "type") === "string" && ["date-time", "date"].includes(_.get(value, "format"))) { + this.updateStoreType(value, _.get(value, "format")); + } + else { + this.updateStoreType(value, _.get(value, "type")); + } + } + } else { + this.updateStoreType(value, _.get(value, "type")); + } + }) + } + recursive(_.get(schema, "properties")) + } + + public mapDataTypes(schema: any,): any { + this.updateMappings(schema); + return schema; + } + +} diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaMapping.json b/api-service/src/v2/services/SchemaGenerateService/SchemaMapping.json new file mode 100644 index 00000000..19843cf5 --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaMapping.json @@ -0,0 +1,103 @@ +{ + "text": { + "arrival_format": ["string"], + "store_format": { + "string": { + "jsonSchema": "string", + "datasource": "string" + }, + "date-time": { + "jsonSchema": "string", + "datasource": "string" + }, + "date": { + "jsonSchema": "string", + "datasource": "string" + }, + "boolean": { + "jsonSchema": "string", + "datasource": "boolean" + }, + "epoch": { + "jsonSchema": "string", + "datasource": "integer" + }, + "long": { + "jsonSchema": "string", + "datasource": "long" + }, + "double": { + "jsonSchema": "string", + "datasource": "double" + }, + "bigdecimal": { + "jsonSchema": "string", + "datasource": "double" + }, + "integer": { + "jsonSchema": "string", + "datasource": "long" + } + } + }, + "number": { + "arrival_format": ["number", "integer"], + "store_format": { + "integer": { + "jsonSchema": "integer", + "datasource": "long" + }, + "float": { + "jsonSchema": "number", + "datasource": "double" + }, + "long": { + "jsonSchema": "integer", + "datasource": "long" + }, + "double": { + "jsonSchema": "number", + "datasource": "double" + }, + "bigdecimal": { + "jsonSchema": "number", + "datasource": "double" + }, + "epoch":{ + "jsonSchema": "integer", + "datasource": "long" + }, + "number": { + "jsonSchema": "number", + "datasource": "double" + } + } + }, + "object": { + "arrival_format": ["object"], + "store_format": { + "object": { + "jsonSchema": "object", + "datasource": "json" + } + } + }, + "array": { + "arrival_format": ["array"], + "store_format": { + "array": { + "jsonSchema": "array", + "datasource": "array" + } + } + }, + "boolean": { + "arrival_format": ["boolean"], + "store_format": { + "boolean": { + "jsonSchema": "boolean", + "datasource": "boolean" + } + } + } +} diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaMerger.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaMerger.ts new file mode 100644 index 00000000..a35cc237 --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaMerger.ts @@ -0,0 +1,18 @@ +import * as _ from "lodash"; + +export class SchemaMerger { + + mergeSchema(schema: Map[]): any { + try { + let data = {}; + _.map(schema, (item: any) => { + data = _.merge(data, item) + }); + return data; + } + catch (error) { + console.log(error) + } + } + +} \ No newline at end of file diff --git a/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts b/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts new file mode 100644 index 00000000..bb87fe32 --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts @@ -0,0 +1,74 @@ +import _ from "lodash" +import { Conflict, ConflictTypes, Suggestion, SuggestionsTemplate } from "../../types/SchemaModel" +import constants from "./Constants.json" +import { SchemaSuggestionTemplate } from "./Template" + +export class SuggestionTemplate { + + public createSuggestionTemplate(sample: ConflictTypes[]): SuggestionsTemplate[] { + return _.map(sample, (value, key) => { + const dataTypeSuggestions = this.getSchemaMessageTemplate(value.schema) + const requiredSuggestions = this.getRequiredMessageTemplate(value.required) + const formatSuggestions = this.getPropertyFormatTemplate(value.formats) + return { + "property": value.absolutePath, + "suggestions": _.reject([dataTypeSuggestions, requiredSuggestions, formatSuggestions], _.isEmpty) + } + }) + } + + private getSchemaMessageTemplate(object: Conflict, arrival_format: boolean = false): Suggestion | [] { + if (_.isEmpty(object)) return {} + let message, arrival_format_message, advice; + if (object.type === constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD) { + message = SchemaSuggestionTemplate.getSchemaNullTypeMessage(object.conflicts, object.property); + advice = SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.NULL_TYPE_PROPERTY.ADVICE; + } else { + let { conflictMessage, arrivalFormatMessage } = SchemaSuggestionTemplate.getSchemaDataTypeMessage(object.conflicts, object.property); + message = conflictMessage; + arrival_format_message = arrivalFormatMessage; + advice = SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.ADVICE; + } + + const suggestion = { + message: message, + advice, + resolutionType: object.resolution["type"], + severity: object.severity, + path: object.path + }; + if (arrival_format && arrival_format_message) { + suggestion["message"] = arrival_format_message; + suggestion["arrivalConflict"] = true; + suggestion["resolutionType"] = constants.SCHEMA_RESOLUTION_TYPE.ARRIVAL_FORMAT; + } else if(arrival_format && !arrival_format_message) + return []; + return suggestion; + } + + + public getPropertyFormatTemplate(object: Conflict): Suggestion { + if (_.isEmpty(object)) return {} + const conflictMessage = SchemaSuggestionTemplate.getSchemaFormatMessage(object.conflicts, object.property) + const adviceObj = SchemaSuggestionTemplate.getSchemaFormatAdvice(object.conflicts) + return { + message: conflictMessage, + advice: adviceObj.advice, + resolutionType: object.resolution["type"], + severity: object.severity, + path: object.path + } + } + + public getRequiredMessageTemplate(object: Conflict): Suggestion { + if (_.isEmpty(object)) return {} + const conflictMessage = SchemaSuggestionTemplate.getSchemaRequiredTypeMessage(object.conflicts, object.property) + return { + message: conflictMessage, + advice: SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.OPTIONAL_PROPERTY.ADVICE, + resolutionType: object.resolution["type"], + severity: object.severity, + path: object.path + } + } +} diff --git a/api-service/src/v2/services/SchemaGenerateService/Template.ts b/api-service/src/v2/services/SchemaGenerateService/Template.ts new file mode 100644 index 00000000..49c86228 --- /dev/null +++ b/api-service/src/v2/services/SchemaGenerateService/Template.ts @@ -0,0 +1,107 @@ +import _ from "lodash"; +import { Conflict } from "../../types/SchemaModel"; +import constants from "./Constants.json"; +import { dataMappingPaths } from "./../SchemaGenerateService/SchemaHandler"; + +export const SchemaSuggestionTemplate = { + TEMPLATES: { + SCHEMA_SUGGESTION: { + CREATE: { + OPTIONAL_PROPERTY: { + MESSAGE: "Conflict in the Schema Generation", + ADVICE: "The Property looks to be Optional. System has updated the property schema to optional", + SEVERITY: constants.SEVERITY.MEDIUM + }, + DATATYPE_PROPERTY: { + MESSAGE: "Conflict in the Schema Generation", + ADVICE: "System can choose highest occurance property or last appeared object property", + SEVERITY: constants.SEVERITY["MUST-FIX"] + }, + NULL_TYPE_PROPERTY: { + MESSAGE: "Conflict in the Schema Generation", + ADVICE: "The Property has a 'null' data type. 'null' is not a valid data type. Please review and update the property schema accordingly.", + SEVERITY: constants.SEVERITY["MUST-FIX"] + }, + FORMAT_PROPERTY: { + MESSAGE: "The Property", + DATE_ADVICE: { + MESSAGE: "The System can index all data on this column", + SEVERITY: constants.SEVERITY.HIGH + }, + DATETIME_ADVICE: { + MESSAGE: "The System can index all data on this column", + SEVERITY: constants.SEVERITY.LOW + }, + UUID_ADVICE: { + MESSAGE: "Suggest to not to index the high cardinal columns", + SEVERITY: constants.SEVERITY.LOW + }, + IPV4_ADVICE: { + MESSAGE: "Suggest to Mask the Personal Information", + SEVERITY: constants.SEVERITY.LOW + }, + IPV6_ADVICE: { + MESSAGE: "Suggest to Mask the Personal Information", + SEVERITY: constants.SEVERITY.LOW + }, + EMAIL_ADVICE: { + MESSAGE: "Suggest to Mask the Personal Information", + SEVERITY: constants.SEVERITY.LOW + } + } + }, + UPDATE: { + REQUIRED_PROPERTY: { + ADVICE: "Might Required to reprocess the existing data" + }, + DATATYPE_PROPERTY: { + ADVICE: "Might Required to reprocess the existing data" + } + } + + } + }, + + getSchemaDataTypeMessage(conflicts: Conflict, property: string) { + const updatedConflicts: any = {}; + _.map((conflicts), (value, key) => { + const path: any = _.get(dataMappingPaths, key); + const types = _.split(path, "."); + if (path && _.size(types) > 0 && !_.keys(updatedConflicts).includes(types[0])) { + updatedConflicts[types[0]] = value; + } + }); + let response: Record = { + conflictMessage: _.template( + `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.MESSAGE} at property: '${property}'. The property type <% _.map(conflicts, (value, key, list) => { %><%= key %>: <%= value %> time(s)<%= _.last(list) === value ? '' : ', ' %><% }); %><%= _.isEmpty(conflicts) ? '' : '' %>`)({ conflicts }), + arrivalFormatMessage: null, + }; + if (_.keys(updatedConflicts).length > 1) { + _.set(response, "arrivalFormatMessage", _.template(`${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.MESSAGE} at property: '${property}'. The property type <% _.map(conflicts, (value, key, list) => { %><%= key %>: <%= value %> time(s)<%= _.last(list) === value ? '' : ', ' %><% }); %><%= _.isEmpty(conflicts) ? '' : '' %>`)({ conflicts: updatedConflicts })); + } + return response; + }, + + getSchemaNullTypeMessage(conflicts: Conflict, property: string): string { + return `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.NULL_TYPE_PROPERTY.MESSAGE} at property: '${property}'. ${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.NULL_TYPE_PROPERTY.ADVICE}`; + }, + + getSchemaFormatMessage(conflicts: Conflict, property: string): string { + const conflictKey = _.keys(conflicts)[0]; + return `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.FORMAT_PROPERTY.MESSAGE} '${property}' appears to be '${conflictKey}' format type.`; + }, + + getSchemaRequiredTypeMessage(conflicts: Conflict, property: string): string { + const message = _.template( + `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.OPTIONAL_PROPERTY.MESSAGE} at property: '${property}'. The property <% _.map(conflicts, (value, key, list) => { %><%= key %>: only <%= value %> time(s) appeared <%= _.last(list) === value ? '' : '' %><% }); %><%= _.isEmpty(conflicts) ? '' : '' %>`)({ conflicts }); + return message + }, + + getSchemaFormatAdvice(conflicts: Conflict): any { + const conflictKey = `${_.upperCase(_.replace(_.keys(conflicts)[0], "-", ""))}_ADVICE`.replace(/\s/g, "") + return { + "advice": _.get(this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.FORMAT_PROPERTY, `${conflictKey}.MESSAGE`), + } + } + +} diff --git a/api-service/src/v2/types/ConfigModels.ts b/api-service/src/v2/types/ConfigModels.ts index 538685aa..320cbb32 100644 --- a/api-service/src/v2/types/ConfigModels.ts +++ b/api-service/src/v2/types/ConfigModels.ts @@ -1,3 +1,6 @@ +import { IngestionConfig } from "./IngestionModels"; +import { IDataSourceRules, IRules } from "./QueryModels"; + export interface ExtractionConfig { is_batch_event: boolean; extraction_key: string; @@ -41,4 +44,17 @@ export interface DatasetConfig { redis_db: number; index_data: boolean; } +export interface DataSetConfig { + querying: IDataSourceRules + indexConfiguration: IngestionConfig, + processing: DatasetProcessing +} +export interface DatasetProcessing { + topic: string; + extraction: ExtractionConfig; + dedup_config: DedupConfig; + validation_config: ValidationConfig; + denorm_config: DenormConfig; + router_config: RouterConfig; +} diff --git a/api-service/src/v2/types/IngestionModels.ts b/api-service/src/v2/types/IngestionModels.ts index 9fc4237b..b43f0805 100644 --- a/api-service/src/v2/types/IngestionModels.ts +++ b/api-service/src/v2/types/IngestionModels.ts @@ -50,3 +50,15 @@ export interface IngestionSchemeRequest { schema: Map[], config: IngestionConfig } + +export interface RollupInfo { + summary?: RollupSuggestionsSummary; +} + +interface RollupSuggestionsSummary { + [key: string]: { + path: string; + cardinality: number; + index: boolean; + }; +} \ No newline at end of file diff --git a/api-service/src/v2/types/SchemaModel.ts b/api-service/src/v2/types/SchemaModel.ts new file mode 100644 index 00000000..eb592d8a --- /dev/null +++ b/api-service/src/v2/types/SchemaModel.ts @@ -0,0 +1,115 @@ +export interface DataSetConfig { + querying: Record + indexConfiguration: Record + processing: Record + } + + +export interface DatasetSchemeRequest { + data: Map[], + config: DatasetSchemaConfig +} + +export interface DatasetSchemaConfig { + dataset: string, + isBatch?: boolean + extractionKey: string, +} + +export interface DatasetSchemaResponse { + schema: any; + configurations: DataSetConfig + dataMappings: Record +} + +export interface DataSetConfig { + querying: Record + indexConfiguration: Record + processing: Record + } + + +export interface DatasetSchemaResponse { + schema: any; + configurations: DataSetConfig + dataMappings: Record +} + +export interface DatasetSchemeRequest { + data: Map[], + config: DatasetSchemaConfig +} + +export interface DatasetSchemaConfig { + dataset: string, + isBatch?: boolean + extractionKey: string, +} + +export interface SuggestionsTemplate { + property: string; + suggestions: Suggestion[]; +} + +export interface Suggestion { + message: string; + arrivalConflict?: boolean; + advice: string; + resolutionType: string; + severity: string; +} + +export interface ConflictTypes { + schema: Conflict; + required: Conflict; + formats: Conflict; + absolutePath: string; +} + +export interface Conflict { + property: string, + type: string, + path: string, + conflicts: any, + values: any[], + resolution: any, + severity: string +} + + +export interface FlattenSchema { + property: string + dataType: string + isRequired: boolean + path: string | any; + absolutePath: string + formate: string +} + +export interface Occurance { + property: { [key: string]: number }; + dataType: { [key: string]: number }; + isRequired: { [key: string]: number }; + path: { [key: string]: number }; + absolutePath: { [key: string]: number }; + format: { [key: string]: number }; +} + +export interface UniqueValues { + [key: string]: any[]; +} + +export interface FieldSchema { + type?: string | any; + format?: string + properties?: Record; + items?: FieldSchema; +} + +export interface RollupSummary { + [key: string]: { + path: string; + cardinality: number; + index: boolean; + }; +} From 72b67b55f06bdcedd1482a98a8a2e679dbb2cc50 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 17 Jul 2024 22:56:52 +0530 Subject: [PATCH 038/235] #OBS-I116: feat: Schema validation fix --- .../GenerateDataSchema/GenerateDataSchema.ts | 5 +- .../RequestValidationSchema.json | 78 ++++++++++++------- 2 files changed, 52 insertions(+), 31 deletions(-) diff --git a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts index a47225c0..309dfe39 100644 --- a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts +++ b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -29,18 +29,17 @@ const validateRequest = async (req: Request) => { const dataSchema = async (req: Request, res: Response) => { await validateRequest(req) - const request = req.body + const request = req.body.request const dataSchemaSpec = schemaGenerate(request.data, request.config) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataSchemaSpec }); } const schemaGenerate = (sample: Map[], config: Record): any => { - const { isBatch = false, extractionKey } = config; + const { isBatch = false, extractionKey, dataset } = config; const isJsonSchema = checkJsonSchema(_.head(sample) || new Map()); const schemaInference = new SchemaInference(); const schemaArrayValidator = new SchemaArrayValidator(); - const dataset = _.get(config, "dataset") if (isJsonSchema) { let result = process(sample, dataset) result.schema = removeNonIndexColumns(result.schema) diff --git a/api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json b/api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json index 0a882400..acffa0ec 100644 --- a/api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json +++ b/api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json @@ -1,34 +1,56 @@ { - "type": "object", - "properties": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasets.dataschema"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { "data": { - "type": "array", - "items": { - "type": "object" - }, - "minItems": 1 + "type": "array", + "items": { + "type": "object" + }, + "minItems": 1 }, "config": { - "type": "object", - "properties": { - "dataset": { - "type": "string" - }, - "isBatch": { - "type": "boolean" - }, - "extractionKey": { - "type": "string" - } + "type": "object", + "properties": { + "dataset": { + "type": "string" }, - "required": [ - "dataset" - ] + "isBatch": { + "type": "boolean" + }, + "extractionKey": { + "type": "string" + } + }, + "required": ["dataset"] } - }, - "required": [ - "data", - "config" - ], - "additionalProperties": false -} \ No newline at end of file + }, + "required": ["data", "config"], + "additionalProperties": false + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false +} From 9703899ad8a73a94fe6e4177485106194bfe6e70 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 18 Jul 2024 18:04:20 +0530 Subject: [PATCH 039/235] #OBS-I126: swagger doc updated --- api-service/swagger-doc/v2_updated_doc_openapi.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml index a29de464..87625317 100644 --- a/api-service/swagger-doc/v2_updated_doc_openapi.yml +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -5,12 +5,14 @@ info: servers: - url: localhost:3000 - url: http://localhost:3007 +tags: + - name: Dataset api's paths: /v2/datasets/create: post: tags: - - Dataset API's - summary: API to create a new dataset. + - Dataset api's + summary: Dataset create requestBody: content: application/json: From 80ac93d3dca669957986d86153dd7d1453b4718d Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sat, 20 Jul 2024 19:42:25 +0530 Subject: [PATCH 040/235] #OBS-I116: Dataset update api test cases --- .../DatasetUpdate/DatasetDenorm.spec.ts | 2 +- .../DatasetTransformation.spec.ts | 152 +---------------- .../DatasetUpdate/DatasetUpdate.spec.ts | 128 ++------------ .../DatasetUpdate/DatasetValidation.spec.ts | 6 +- .../DatasetUpdate/Fixtures.ts | 156 +++++------------- 5 files changed, 57 insertions(+), 387 deletions(-) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index aaddfd4a..c0f21815 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -1,5 +1,5 @@ import app from "../../../../app"; -import chai, { expect } from "chai"; +import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts index 14ae7c5a..6d38059c 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts @@ -7,7 +7,6 @@ import { describe, it } from 'mocha'; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; -import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" chai.use(spies); @@ -23,15 +22,9 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { it("Success: Dataset transformations successfully added", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", "transformations_config":[] }) }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([]) - }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) @@ -55,15 +48,9 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { it("Success: Dataset transformations successfully removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", "transformations_config":[{ "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }] }) }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) @@ -84,53 +71,9 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { }); }); - it("Success: Dataset transformations successfully updated", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_TRANSFORMATIONS_UPDATE) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - it("Success: When payload contains same transformation field_key to be added, updated or removed", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) @@ -152,93 +95,4 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { }); }); - it("Failure: When transformation fields provided to add already exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key1" }, { field_key: "key3" }]) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset transformations already exists") - res.body.error.code.should.be.eq("DATASET_TRANSFORMATIONS_EXIST") - done(); - }); - }); - - it("Failure: When transformation fields provided to update do not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset" , version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key7" }, { field_key: "key2" }]) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset transformations do not exist to update") - res.body.error.code.should.be.eq("DATASET_TRANSFORMATIONS_DO_NOT_EXIST") - done(); - }); - }); - - it("Failure: When transformation fields provided to remove do not exists", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", type:"dataset", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { - denorm_fields: [{ - "denorm_key": "actor.id", - "denorm_out_field": "mid" - }] - } - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key7" }, { field_key: "key3" }]) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset transformations do not exist to remove") - res.body.error.code.should.be.eq("DATASET_TRANSFORMATIONS_DO_NOT_EXIST") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index d4d2f921..8ec55ff6 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -8,10 +8,9 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; -import { apiId, errorCode, invalidInputErrCode } from "../../../controllers/DatasetUpdate/DatasetUpdate" +import { apiId, invalidInputErrCode } from "../../../controllers/DatasetUpdate/DatasetUpdate" import { DatasourceDraft } from "../../../models/DatasourceDraft"; -import { Dataset } from "../../../models/Dataset"; -import { DatasetTransformations } from "../../../models/Transformation"; + chai.use(spies); chai.should(); @@ -25,7 +24,7 @@ describe("DATASET UPDATE API", () => { it("Dataset updation success: When minimal request payload provided", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" }) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) @@ -50,44 +49,14 @@ describe("DATASET UPDATE API", () => { it("Dataset updation success: When full request payload provided", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", type: "dataset", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { + id: "telemetry", status: "Draft", type: "event", version_key: validVersionKey, tags: ["tag1", "tag2"], denorm_config: { denorm_fields: [{ "denorm_key": "actor.id", "denorm_out_field": "mid" }] - } - }) - }) - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", - "properties": { - "eid": { "type": "string" }, - "ets": { "type": "string" } - }, - "additionalProperties": true - }, + }, api_version: "v2" }) }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasetTransformationsDraft, "bulkCreate", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetTransformationsDraft, "destroy", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasourceDraft, "update", () => { - return Promise.resolve({}) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) @@ -141,7 +110,7 @@ describe("DATASET UPDATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset does not exists to update") + res.body.error.message.should.be.eq("Dataset does not exists with id:telemetry") res.body.error.code.should.be.eq("DATASET_NOT_EXISTS") done(); }); @@ -149,7 +118,7 @@ describe("DATASET UPDATE API", () => { it("Dataset updation failure: When dataset to update is outdated", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: "1813444815918", api_version: "v2" }) + return Promise.resolve({ id: "telemetry", type:"event", status: "Draft", version_key: "1813444815918", api_version: "v2" }) }) chai @@ -170,7 +139,7 @@ describe("DATASET UPDATE API", () => { it("Dataset updation failure: Dataset to update is not in draft state", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ status: "Live" }) + return Promise.resolve({ id: "telemetry", type:"event", status: "Live", version_key: "1713444815918", api_version: "v2" }) }) chai @@ -203,8 +172,7 @@ describe("DATASET UPDATE API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to update dataset") - res.body.error.code.should.be.eq(errorCode) + res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR") done(); }); }); @@ -217,7 +185,7 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset name updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" }) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) @@ -267,39 +235,7 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset data schema updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" - }) - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai.spy.on(DatasourceDraft, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetDraft, "update", () => { - return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_UPDATE_DATA_SCHEMA_VALID) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("SUCCESS") - res.body.params.msgid.should.be.eq(msgid) - res.body.result.id.should.be.eq("telemetry") - res.body.result.message.should.be.eq("Dataset is updated successfully") - res.body.result.version_key.should.be.a("string") - done(); - }); - }); - - it("Success: Ingestion spec updateded successfully", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetTransformationsDraft, "findAll", () => { @@ -328,46 +264,6 @@ describe("DATASET UPDATE API", () => { }); }); - it("Failure: When timestamp key does not exist in the data schema", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" - }) - }) - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", - "properties": { - "eid": { "type": "string" }, - "ets": { "type": "string" } - }, - "additionalProperties": true - }, - }) - }) - chai.spy.on(DatasetTransformations, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) - chai - .request(app) - .patch("/v2/datasets/update") - .send(TestInputsForDatasetUpdate.DATASET_WITH_INVALID_TIMESTAMP) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.id.should.be.eq(apiId); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Provided timestamp key not found in the data schema") - res.body.error.code.should.be.eq("DATASET_TIMESTAMP_NOT_FOUND") - done(); - }); - }); - it("Failure: Failed to update data schema", (done) => { chai .request(app) @@ -395,7 +291,7 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset config updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type: "dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts index a56444ae..1fc1aaed 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts @@ -22,7 +22,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { it("Success: Dataset validation configs updated when validation is true", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -48,7 +48,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { it("Success: Dataset validation configs updated with default values when validation is false", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve({ - id: "telemetry", status: "Draft", version_key: validVersionKey, type:"dataset" + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { @@ -57,7 +57,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { chai .request(app) .patch("/v2/datasets/update") - .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, validation_config: { "validate": false } } }) + .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, validation_config: { "validate": false, "mode": "Strict" } } }) .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index 4e7e9b10..eb2c683b 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -25,7 +25,7 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": "tag1", + "value": "tag1", "action": "upsert" }] } @@ -37,7 +37,7 @@ export const TestInputsForDatasetUpdate = { "version_key": validVersionKey, "tags": [ { - "values": "tag1", + "value": "tag1", "action": "remove" }] } @@ -84,16 +84,7 @@ export const TestInputsForDatasetUpdate = { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "upsert" - }] + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }], } }, @@ -193,8 +184,18 @@ export const TestInputsForDatasetUpdate = { "dataset_id": "telemetry", "version_key": validVersionKey, "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets" + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets", + "data_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] } } }, @@ -203,16 +204,7 @@ export const TestInputsForDatasetUpdate = { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" - }] + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }], } }, @@ -277,62 +269,45 @@ export const TestInputsForDatasetUpdate = { "denorm_config": { "denorm_fields": [ { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "userdata" + "denorm_out_field": "userdata", + "dataset_id": "master" }, "action": "upsert" }, { - "values": { + "value": { "denorm_key": "actor.id", - "denorm_out_field": "mid" + "denorm_out_field": "mid", + "dataset_id": "master" }, "action": "remove" } ] }, - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "upsert" + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }, { "value": { "field_key": "key2", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "remove" }], + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false }, - { - "values": { - "field_key": "key2", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" + "keys_config": { + "timestamp_key": "ets", + "data_key": "ets" }, - { - "values": { - "field_key": "key3", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "update" - } - ], - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "file_upload_path": ["/config/file.json"] + "file_upload_path": [ + "telemetry.json" + ] }, "tags": [ { - "values": "tag1", + "value": "tag1", "action": "remove" }, { - "values": "tag3", + "value": "tag3", "action": "upsert" } ] @@ -363,7 +338,7 @@ export const TestInputsForDatasetUpdate = { } } }, - + DATASET_UPDATE_WITH_SAME_DENORM_REMOVE: { ...requestStructure, request: { "dataset_id": "telemetry", @@ -397,62 +372,7 @@ export const TestInputsForDatasetUpdate = { "dataset_id": "telemetry", "version_key": validVersionKey, "name": "sb-telemetry", - "transformation_config": [ - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "upsert" - }, - { - "values": { - "field_key": "key1", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "upsert" - }, - { - "values": { - "field_key": "key2", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" - }, - { - "values": { - "field_key": "key2", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "remove" - }, - { - "values": { - "field_key": "key3", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "update" - }, - { - "values": { - "field_key": "key3", - "transformation_function": {}, - "mode": "Strict", - "metadata": {} - }, - "action": "update" - } - ] + "transformations_config": [{ "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }, { "value": { "field_key": "key1", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }, "action": "upsert" }] } } } From 8d284ece247751a88c562d9ea278b8f1b263f0f8 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sat, 20 Jul 2024 20:02:45 +0530 Subject: [PATCH 041/235] #OBS-I116: fix: linting fixes --- .../v2/controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition.ts | 22 +++---- .../DatasetUpdate/DatasetUpdate.ts | 4 +- .../GenerateDataSchema/GenerateDataSchema.ts | 22 +++---- .../exceptions/SchemaGenerationException.ts | 2 +- api-service/src/v2/middlewares/errors.ts | 6 +- api-service/src/v2/services/CipherService.ts | 10 +-- api-service/src/v2/services/DatasetService.ts | 12 ++-- .../SchemaGenerateService/ConfigSuggester.ts | 4 +- .../DataSchemaService.ts | 66 +++++++++---------- .../SchemaGenerateService/SchemaAnalyser.ts | 40 +++++------ .../SchemaArrayValidator.ts | 14 ++-- .../SchemaGeneratorUtils.ts | 6 +- .../SchemaGenerateService/SchemaHandler.ts | 42 ++++++------ .../SuggestionTemplate.ts | 4 +- .../SchemaGenerateService/Template.ts | 2 +- api-service/src/v2/services/TableGenerator.ts | 46 ++++++------- api-service/src/v2/types/ConfigModels.ts | 2 +- 18 files changed, 154 insertions(+), 152 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts index c82008fc..9cd64d8f 100644 --- a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts @@ -16,7 +16,7 @@ const validateRequest = (req: Request) => { const { dataset_id } = req.params; const fields = req.query.fields; - if(fields && typeof fields !== 'string') { + if(fields && typeof fields !== "string") { throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS_VAL", `The specified fields [${fields}] in the query param is not a string.`, "BAD_REQUEST", 400); } const fieldValues = fields ? _.split(fields, ",") : []; diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 948a7772..dac2daeb 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -102,7 +102,7 @@ const readyForPublish = async (dataset: Record) => { statusCode: 400 } } - _.set(draftDataset, 'status', DatasetStatus.ReadyToPublish) + _.set(draftDataset, "status", DatasetStatus.ReadyToPublish) await datasetService.updateDraftDataset(draftDataset) } @@ -129,7 +129,7 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) // 1. Check if there are denorm fields and dependent master datasets are published const denormConfig = _.get(draftDataset, "denorm_config") if(denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { - const datasetIds = _.map(denormConfig.denorm_fields, 'dataset_id') + const datasetIds = _.map(denormConfig.denorm_fields, "dataset_id") if(_.includes(datasetIds, draftDataset.id)) { throw { code: "SELF_REFERENCING_MASTER_DATA", @@ -141,22 +141,22 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) const masterDatasets = await datasetService.findDatasets({id: datasetIds, type: "master"}, ["id", "status", "dataset_config", "api_version"]) const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) - let datasetStatus : Record = { + const datasetStatus : Record = { dataset_id: denormField.dataset_id, exists: (md) ? true : false, isLive: (md) ? md.status === "Live" : false, status: md.status } if(md.api_version === "v2") - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); + datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); else - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); + datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); return datasetStatus; }) const invalidMasters = _.filter(masterDatasetsStatus, {isLive: false}) if(_.size(invalidMasters) > 0) { - const invalidIds = _.map(invalidMasters, 'dataset_id') + const invalidIds = _.map(invalidMasters, "dataset_id") throw { code: "DEPENDENT_MASTER_DATA_NOT_LIVE", message: `The datasets with id:${invalidIds} are not in published status`, @@ -169,13 +169,13 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) draftDataset["denorm_config"] = { redis_db_host: defaultDatasetConfig.denorm_config.redis_db_host, redis_db_port: defaultDatasetConfig.denorm_config.redis_db_port, - denorm_fields: _.map(masterDatasetsStatus, 'denorm_field') + denorm_fields: _.map(masterDatasetsStatus, "denorm_field") } } } const updateMasterDataConfig = async (draftDataset: Record) => { - if(draftDataset.type === 'master') { + if(draftDataset.type === "master") { if(draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() if(_.isEmpty(results)) { @@ -187,7 +187,7 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } const nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; - _.set(draftDataset, 'dataset_config.cache_config.redis_db', nextRedisDB) + _.set(draftDataset, "dataset_config.cache_config.redis_db", nextRedisDB) } } } @@ -208,13 +208,13 @@ const canRetireIfMasterDataset = async (dataset: Record) => { const draftDatasets = await datasetService.findDraftDatasets({ status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, ["denorm_config", "id", "status"]) || [] const allDatasets = _.union(liveDatasets, draftDatasets) const extractDenormFields = _.map(allDatasets, function(depDataset) { - return {dataset_id: _.get(depDataset, 'id'), status: _.get(depDataset, 'status'), denorm_datasets: _.map(_.get(depDataset, 'denorm_config.denorm_fields'), 'dataset_id')} + return {dataset_id: _.get(depDataset, "id"), status: _.get(depDataset, "status"), denorm_datasets: _.map(_.get(depDataset, "denorm_config.denorm_fields"), "dataset_id")} }) const deps = _.filter(extractDenormFields, function(depDS) { return _.includes(depDS.denorm_datasets, dataset.id)}) if (_.size(deps) > 0) { const denormErrMsg = `Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset` - throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function(o) { return _.omit(o, 'denorm_datasets')})) + throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function(o) { return _.omit(o, "denorm_datasets")})) } } } diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts index 1a8d68a5..e5384354 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts @@ -9,6 +9,7 @@ import { datasetService } from "../../services/DatasetService"; import { schemaValidation } from "../../services/ValidationService"; import DatasetUpdate from "./DatasetUpdateValidationSchema.json"; import { obsrvError } from "../../types/ObsrvError"; +import logger from "../../logger"; export const apiId = "api.datasets.update"; export const invalidInputErrCode = "DATASET_UPDATE_INPUT_INVALID" @@ -25,6 +26,7 @@ const validateRequest = async (req: Request) => { const datasetBody = req.body.request const { dataset_id, version_key, ...rest } = datasetBody if (_.isEmpty(rest)) { + logger.error({ apiId, message: `Provide atleast one field in addition to the dataset_id:${dataset_id} and version_key:${version_key} to update the dataset` }) throw obsrvError(datasetId, "DATASET_UPDATE_NO_FIELDS", "Provide atleast one field in addition to the dataset_id to update the dataset", "BAD_REQUEST", 400) } @@ -63,7 +65,7 @@ const datasetUpdate = async (req: Request, res: Response) => { const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any): Record => { - let dataset: Record = { + const dataset: Record = { version_key: Date.now().toString(), name: datasetReq.name || _.get(datasetModel, ["name"]), id: _.get(datasetModel, ["id"]) diff --git a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts index 309dfe39..3d16f5b2 100644 --- a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts +++ b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -41,7 +41,7 @@ const schemaGenerate = (sample: Map[], config: Record) const schemaInference = new SchemaInference(); const schemaArrayValidator = new SchemaArrayValidator(); if (isJsonSchema) { - let result = process(sample, dataset) + const result = process(sample, dataset) result.schema = removeNonIndexColumns(result.schema) result.schema = removeFormats(result.schema) return result @@ -50,7 +50,7 @@ const schemaGenerate = (sample: Map[], config: Record) schema = schemaArrayValidator.validate(schema) const schemaCardinalityAnalyser = new SchemaCardinalityAnalyser(sample, schema) rollupInfo = schemaCardinalityAnalyser.analyse() - let result = process(schema, dataset) + const result = process(schema, dataset) result.schema = removeNonIndexColumns(result.schema) result.schema = removeFormats(result.schema) return result @@ -81,14 +81,14 @@ const checkJsonSchema = (sample: Map): boolean => { const removeNonIndexColumns = (schema: any) => { if (schema.properties) { Object.entries(schema.properties).map(([key, property]: any) => { - _.unset(schema, 'required'); + _.unset(schema, "required"); removeNonIndexColumns(property) }); } else if (schema.items) { removeNonIndexColumns(schema.items) } if (Array.isArray(schema.required) && schema.required.length === 0) { - _.unset(schema, 'required'); + _.unset(schema, "required"); } return schema } @@ -97,17 +97,17 @@ const removeFormats = (schema: any) => { if (schema.properties) { Object.entries(schema.properties).map(([key, property]: any) => { // Removing format to avoid schema validation issues - const isDateTypeField = ['date-time', 'date', 'epoch'].includes((property as any).format); - if (isDateTypeField && _.get(property, 'data_type') === 'string') { - _.set(property, 'data_type', _.get(property, 'format')); - } else if (isDateTypeField && _.get(property, 'data_type') === 'integer') { - _.set(property, 'data_type', 'epoch'); + const isDateTypeField = ["date-time", "date", "epoch"].includes((property as any).format); + if (isDateTypeField && _.get(property, "data_type") === "string") { + _.set(property, "data_type", _.get(property, "format")); + } else if (isDateTypeField && _.get(property, "data_type") === "integer") { + _.set(property, "data_type", "epoch"); } - _.unset(property, 'format'); + _.unset(property, "format"); removeFormats(property) }); } else if (schema.items) { - _.unset(schema.items, 'format'); + _.unset(schema.items, "format"); removeFormats(schema.items) } return schema diff --git a/api-service/src/v2/exceptions/SchemaGenerationException.ts b/api-service/src/v2/exceptions/SchemaGenerationException.ts index 44805da4..fc545e81 100644 --- a/api-service/src/v2/exceptions/SchemaGenerationException.ts +++ b/api-service/src/v2/exceptions/SchemaGenerationException.ts @@ -2,7 +2,7 @@ export class SchemaGenerationException extends Error { statusCode: number; constructor(message: string, code: number) { super(message); - this.name = 'SchemaGenerationException'; + this.name = "SchemaGenerationException"; this.statusCode = code; } } \ No newline at end of file diff --git a/api-service/src/v2/middlewares/errors.ts b/api-service/src/v2/middlewares/errors.ts index 568aaa67..2ca96caf 100644 --- a/api-service/src/v2/middlewares/errors.ts +++ b/api-service/src/v2/middlewares/errors.ts @@ -4,15 +4,15 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; -export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => { +export const errorHandler = (err: Error, req: Request, res: Response) => { logger.error({ path: req.url, req: req.body , ...err }) - let errorMessage = {name: err.name, message: err.message}; + const errorMessage = {name: err.name, message: err.message}; ResponseHandler.errorResponse(errorMessage, req, res); }; -export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, next: NextFunction) => { +export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response) => { logger.error({ path: req.url, req: req.body, resmsgid: _.get(res, "resmsgid") , ...obsrvErr }) ResponseHandler.obsrvErrorResponse(obsrvErr, req, res); diff --git a/api-service/src/v2/services/CipherService.ts b/api-service/src/v2/services/CipherService.ts index b0b86c25..5f85198d 100644 --- a/api-service/src/v2/services/CipherService.ts +++ b/api-service/src/v2/services/CipherService.ts @@ -1,5 +1,5 @@ -import crypto from 'crypto'; -import { config } from '../configs/Config'; +import crypto from "crypto"; +import { config } from "../configs/Config"; class CipherService { public encrypt(data: string) { @@ -8,10 +8,10 @@ class CipherService { config.encryption_config.encryption_key, "", ) - const toEncrypt = Buffer.from(data, 'utf8'); + const toEncrypt = Buffer.from(data, "utf8"); let encryptedString = cipher.update(toEncrypt); encryptedString = Buffer.concat([encryptedString, cipher.final()]) - return encryptedString.toString('base64'); + return encryptedString.toString("base64"); } public decrypt(data: string) { @@ -20,7 +20,7 @@ class CipherService { config.encryption_config.encryption_key, "", ) - const encryptedText = Buffer.from(data, 'base64'); + const encryptedText = Buffer.from(data, "base64"); let decryptedString = decipher.update(encryptedText); decryptedString = Buffer.concat([decryptedString, decipher.final()]) return decryptedString.toString(); diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 43bbcdf0..9de5bc6b 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -94,7 +94,7 @@ class DatasetService { migrateDraftDataset = async (datasetId: string, dataset: Model): Promise => { - let draftDataset : Record = { + const draftDataset : Record = { api_version: "v2" } const dataset_config:any = _.get(dataset, "dataset_config"); @@ -149,7 +149,7 @@ class DatasetService { createDraftDatasetFromLive = async (dataset: Model) => { - let draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); + const draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); const dataset_config:any = _.get(dataset, "dataset_config"); const api_version:any = _.get(dataset, "api_version"); if(api_version === "v1") { @@ -272,7 +272,7 @@ class DatasetService { const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); - _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + _.set(draftDatasource, "ingestion_spec", ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } @@ -281,7 +281,7 @@ class DatasetService { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); - _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + _.set(draftDatasource, "ingestion_spec", ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } @@ -292,7 +292,7 @@ class DatasetService { const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource.ingestion_spec, allFields, draftDatasource.datasource_ref); - _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + _.set(draftDatasource, "ingestion_spec", ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } @@ -300,7 +300,7 @@ class DatasetService { const datasource = _.join([draftDataset.dataset_id,"events"], "_") return { - id: _.join([datasource,type], '_'), + id: _.join([datasource,type], "_"), datasource: draftDataset.dataset_id, dataset_id: draftDataset.dataset_id, datasource_ref: datasource, diff --git a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts index 14a006d4..0eb13bbb 100644 --- a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts +++ b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts @@ -43,8 +43,8 @@ export class ConfigSuggestor { private processingConfig(conflicts: ConflictTypes[]): any { let dedupKeys = _.filter(conflicts, (o) => _.upperCase(o.formats.resolution["type"]) === "DEDUP").map(v => v.formats.property) let matchedDedupFields = [] - let dedupOrderProperty: string = "cardinality" - let dedupOrder: any = "desc" + const dedupOrderProperty: string = "cardinality" + const dedupOrder: any = "desc" if (!_.isUndefined(this.rollupInfo.summary)) { for (const key of Object.keys(this.rollupInfo.summary)) { if (!this.rollupInfo.summary[key].index) { diff --git a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts index 8a77344d..404ad24b 100644 --- a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts +++ b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts @@ -5,34 +5,34 @@ import moment from "moment"; import { SchemaGenerationException } from "../../exceptions/SchemaGenerationException"; const DATE_FORMATS = [ - 'MM/DD/YYYY','DD/MM/YYYY', 'YYYY-MM-DD', 'YYYY-DD-MM', 'YYYY/MM/DD', - 'DD-MM-YYYY', 'MM-DD-YYYY', 'MM-DD-YYYY HH:mm:ss', 'YYYY/MM/DD HH:mm:ss', - 'YYYY-MM-DD HH:mm:ss', 'YYYY-DD-MM HH:mm:ss', 'DD/MM/YYYY HH:mm:ss', - 'DD-MM-YYYY HH:mm:ss', 'MM-DD-YYYY HH:mm:ss.SSS', 'YYYY-MM-DD HH:mm:ss.SSS', - 'YYYY-DD-MM HH:mm:ss.SSS', 'YYYY/MM/DD HH:mm:ss.SSS', 'DD/MM/YYYY HH:mm:ss.SSS', - 'DD-MM-YYYY HH:mm:ss.SSS', 'DD-MM-YYYYTHH:mm:ss.SSSZ', 'YYYY-MM-DDTHH:mm:ss.SSSZ', - 'YYYY-DD-MMTHH:mm:ss.SSSZ', 'YYYY/MM/DDTHH:mm:ss.SSSZ', 'DD/MM/YYYYTHH:mm:ss.SSSZ', - 'YYYY-DD-MMTHH:mm:ss.SSS', 'YYYY/MM/DDTHH:mm:ss.SSS', 'DD/MM/YYYYTHH:mm:ss.SSS', - 'MM-DD-YYYYTHH:mm:ss.SSSZ', 'DD-MM-YYYYTHH:mm:ssZ', 'YYYY-MM-DDTHH:mm:ssZ', - 'YYYY-DD-MMTHH:mm:ssZ', 'YYYY/MM/DDTHH:mm:ssZ', 'DD/MM/YYYYTHH:mm:ssZ', 'MM-DD-YYYYTHH:mm:ssZ', - 'MM-DD-YYYYTHH:mm:ss', 'DD-MM-YYYYTHH:mm:ss', 'YYYY-MM-DDTHH:mm:ss', 'YYYY-DD-MMTHH:mm:ss', - 'YYYY/MM/DDTHH:mm:ss', 'DD/MM/YYYYTHH:mm:ss', 'DD-MM-YYYY HH:mm:ss.SSSZ', 'YYYY-MM-DD HH:mm:ss.SSSZ', - 'YYYY-DD-MM HH:mm:ss.SSSZ', 'YYYY/MM/DD HH:mm:ss.SSSZ', 'DD/MM/YYYY HH:mm:ss.SSSZ', - 'MM-DD-YYYY HH:mm:ss.SSSZ', 'DD-MM-YYYY HH:mm:ssZ', 'YYYY-MM-DD HH:mm:ssZ', 'YYYY-DD-MM HH:mm:ssZ', - 'YYYY/MM/DD HH:mm:ssZ', 'DD/MM/YYYY HH:mm:ssZ', 'MM-DD-YYYY HH:mm:ssZ', 'DD-MM-YYYYTHH:mm:ss.SSSSSSZ', - 'YYYY-MM-DDTHH:mm:ss.SSSSSSZ', 'YYYY-DD-MMTHH:mm:ss.SSSSSSZ', 'YYYY/MM/DDTHH:mm:ss.SSSSSSZ', - 'DD/MM/YYYYTHH:mm:ss.SSSSSSZ', 'MM-DD-YYYYTHH:mm:ss.SSSSSSZ', 'DD/MM/YYYYTHH:mm:ss.SSSSSS', - 'YYYY-DD-MMTHH:mm:ss.SSSSSS', 'YYYY/MM/DDTHH:mm:ss.SSSSSS', 'YYYY-MM-DDTHH:mm:ss.SSSSSS', - 'MM-DD-YYYYTHH:mm:ss.SSSSSS', 'DD-MM-YYYYTHH:mm:ss.SSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSS', - 'YYYY-MM-DD HH:mm:ss.SSSSSS', 'YYYY-DD-MM HH:mm:ss.SSSSSS', 'YYYY/MM/DD HH:mm:ss.SSSSSS', - 'DD/MM/YYYY HH:mm:ss.SSSSSS', 'MM-DD-YYYY HH:mm:ss.SSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSSZ', - 'YYYY-MM-DDTHH:mm:ss.SSSSSSSSSZ', 'YYYY-DD-MMTHH:mm:ss.SSSSSSSSSZ', 'YYYY/MM/DDTHH:mm:ss.SSSSSSSSSZ', - 'DD/MM/YYYYTHH:mm:ss.SSSSSSSSSZ', 'MM-DD-YYYYTHH:mm:ss.SSSSSSSSSZ', 'DD/MM/YYYYTHH:mm:ss.SSSSSSSSS', - 'YYYY-DD-MMTHH:mm:ss.SSSSSSSSS', 'YYYY/MM/DDTHH:mm:ss.SSSSSSSSS', 'YYYY-MM-DDTHH:mm:ss.SSSSSSSSS', - 'MM-DD-YYYYTHH:mm:ss.SSSSSSSSS', 'DD-MM-YYYYTHH:mm:ss.SSSSSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSSSSS', - 'YYYY-MM-DD HH:mm:ss.SSSSSSSSS', 'YYYY-DD-MM HH:mm:ss.SSSSSSSSS', 'YYYY/MM/DD HH:mm:ss.SSSSSSSSS', - 'DD/MM/YYYY HH:mm:ss.SSSSSSSSS', 'MM-DD-YYYY HH:mm:ss.SSSSSSSSS', 'DD-MM-YYYY HH:mm:ss.SSSSSSSSSZ', - 'DD-MM-YYYYTHH:mm:ss.SSSSSSSSSZ', + "MM/DD/YYYY","DD/MM/YYYY", "YYYY-MM-DD", "YYYY-DD-MM", "YYYY/MM/DD", + "DD-MM-YYYY", "MM-DD-YYYY", "MM-DD-YYYY HH:mm:ss", "YYYY/MM/DD HH:mm:ss", + "YYYY-MM-DD HH:mm:ss", "YYYY-DD-MM HH:mm:ss", "DD/MM/YYYY HH:mm:ss", + "DD-MM-YYYY HH:mm:ss", "MM-DD-YYYY HH:mm:ss.SSS", "YYYY-MM-DD HH:mm:ss.SSS", + "YYYY-DD-MM HH:mm:ss.SSS", "YYYY/MM/DD HH:mm:ss.SSS", "DD/MM/YYYY HH:mm:ss.SSS", + "DD-MM-YYYY HH:mm:ss.SSS", "DD-MM-YYYYTHH:mm:ss.SSSZ", "YYYY-MM-DDTHH:mm:ss.SSSZ", + "YYYY-DD-MMTHH:mm:ss.SSSZ", "YYYY/MM/DDTHH:mm:ss.SSSZ", "DD/MM/YYYYTHH:mm:ss.SSSZ", + "YYYY-DD-MMTHH:mm:ss.SSS", "YYYY/MM/DDTHH:mm:ss.SSS", "DD/MM/YYYYTHH:mm:ss.SSS", + "MM-DD-YYYYTHH:mm:ss.SSSZ", "DD-MM-YYYYTHH:mm:ssZ", "YYYY-MM-DDTHH:mm:ssZ", + "YYYY-DD-MMTHH:mm:ssZ", "YYYY/MM/DDTHH:mm:ssZ", "DD/MM/YYYYTHH:mm:ssZ", "MM-DD-YYYYTHH:mm:ssZ", + "MM-DD-YYYYTHH:mm:ss", "DD-MM-YYYYTHH:mm:ss", "YYYY-MM-DDTHH:mm:ss", "YYYY-DD-MMTHH:mm:ss", + "YYYY/MM/DDTHH:mm:ss", "DD/MM/YYYYTHH:mm:ss", "DD-MM-YYYY HH:mm:ss.SSSZ", "YYYY-MM-DD HH:mm:ss.SSSZ", + "YYYY-DD-MM HH:mm:ss.SSSZ", "YYYY/MM/DD HH:mm:ss.SSSZ", "DD/MM/YYYY HH:mm:ss.SSSZ", + "MM-DD-YYYY HH:mm:ss.SSSZ", "DD-MM-YYYY HH:mm:ssZ", "YYYY-MM-DD HH:mm:ssZ", "YYYY-DD-MM HH:mm:ssZ", + "YYYY/MM/DD HH:mm:ssZ", "DD/MM/YYYY HH:mm:ssZ", "MM-DD-YYYY HH:mm:ssZ", "DD-MM-YYYYTHH:mm:ss.SSSSSSZ", + "YYYY-MM-DDTHH:mm:ss.SSSSSSZ", "YYYY-DD-MMTHH:mm:ss.SSSSSSZ", "YYYY/MM/DDTHH:mm:ss.SSSSSSZ", + "DD/MM/YYYYTHH:mm:ss.SSSSSSZ", "MM-DD-YYYYTHH:mm:ss.SSSSSSZ", "DD/MM/YYYYTHH:mm:ss.SSSSSS", + "YYYY-DD-MMTHH:mm:ss.SSSSSS", "YYYY/MM/DDTHH:mm:ss.SSSSSS", "YYYY-MM-DDTHH:mm:ss.SSSSSS", + "MM-DD-YYYYTHH:mm:ss.SSSSSS", "DD-MM-YYYYTHH:mm:ss.SSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSS", + "YYYY-MM-DD HH:mm:ss.SSSSSS", "YYYY-DD-MM HH:mm:ss.SSSSSS", "YYYY/MM/DD HH:mm:ss.SSSSSS", + "DD/MM/YYYY HH:mm:ss.SSSSSS", "MM-DD-YYYY HH:mm:ss.SSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSSZ", + "YYYY-MM-DDTHH:mm:ss.SSSSSSSSSZ", "YYYY-DD-MMTHH:mm:ss.SSSSSSSSSZ", "YYYY/MM/DDTHH:mm:ss.SSSSSSSSSZ", + "DD/MM/YYYYTHH:mm:ss.SSSSSSSSSZ", "MM-DD-YYYYTHH:mm:ss.SSSSSSSSSZ", "DD/MM/YYYYTHH:mm:ss.SSSSSSSSS", + "YYYY-DD-MMTHH:mm:ss.SSSSSSSSS", "YYYY/MM/DDTHH:mm:ss.SSSSSSSSS", "YYYY-MM-DDTHH:mm:ss.SSSSSSSSS", + "MM-DD-YYYYTHH:mm:ss.SSSSSSSSS", "DD-MM-YYYYTHH:mm:ss.SSSSSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSSSSS", + "YYYY-MM-DD HH:mm:ss.SSSSSSSSS", "YYYY-DD-MM HH:mm:ss.SSSSSSSSS", "YYYY/MM/DD HH:mm:ss.SSSSSSSSS", + "DD/MM/YYYY HH:mm:ss.SSSSSSSSS", "MM-DD-YYYY HH:mm:ss.SSSSSSSSS", "DD-MM-YYYY HH:mm:ss.SSSSSSSSSZ", + "DD-MM-YYYYTHH:mm:ss.SSSSSSSSSZ", ]; export class SchemaInference { @@ -49,17 +49,17 @@ export class SchemaInference { if (extracted) { return this.inferSchema(extracted); } else { - throw new SchemaGenerationException('Unable to extract the batch data.', httpStatus.BAD_REQUEST); + throw new SchemaGenerationException("Unable to extract the batch data.", httpStatus.BAD_REQUEST); } } else { - throw new SchemaGenerationException('Extraction key not found.', httpStatus.BAD_REQUEST); + throw new SchemaGenerationException("Extraction key not found.", httpStatus.BAD_REQUEST); } }) } private validateEpoch(schema: any, sample: any, path: any) { Object.entries(sample).map(([key, value]) => { - if (value && typeof value == 'object') { + if (value && typeof value == "object") { this.validateEpoch(schema, value, `${path}.${key}.properties`) } const { isValidTimestamp, type } = this.isValidTimestamp(value); @@ -77,7 +77,7 @@ export class SchemaInference { isValidTimestamp(value: any) { const dataType = typeof value; switch (dataType) { - case 'string': + case "string": const epochRegex = /^\d+$/ig; if(epochRegex.test(value)){ const parsedValue = parseInt(value, 10); @@ -90,7 +90,7 @@ export class SchemaInference { isValidTimestamp: moment(value, DATE_FORMATS, true).isValid(), type: "date-time" } - case 'number': + case "number": // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds return { isValidTimestamp: value >= 946684800 && moment(value).isValid(), diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts index 14ee0e3d..4f0cb3ce 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts @@ -25,7 +25,7 @@ export class SchemaAnalyser { const result: FlattenSchema[] = _.flatten(this.schemas.map(element => { return this.flattenSchema(new Map(Object.entries(element))); })) - const conflicts = Object.entries(_.groupBy(result, 'path')).map(([key, value]) => { + const conflicts = Object.entries(_.groupBy(result, "path")).map(([key, value]) => { return this.getSchemaConflictTypes(this.getOccurance(value, key)) }) return _.filter(conflicts, obj => (!_.isEmpty(obj.schema) || !_.isEmpty(obj.required) || !_.isEmpty(obj.formats))) @@ -43,7 +43,7 @@ export class SchemaAnalyser { let schemaConflicts = this.findDataTypeConflicts(occuranceObj,) const requiredConflicts = (_.size(this.schemas) > this.minimumSchemas) ? this.findOptionalPropConflicts(occuranceObj) : {} const formatConflict = this.findFormatConflicts(occuranceObj) - if(_.size(_.keys(schemaConflicts)) > 0) { + if (_.size(_.keys(schemaConflicts)) > 0) { schemaConflicts = { ...schemaConflicts, path: updatedPath } } return { "schema": schemaConflicts, "required": requiredConflicts, "formats": formatConflict, "absolutePath": updatedPath } @@ -53,14 +53,14 @@ export class SchemaAnalyser { * Method to get the data type conflicts */ private findDataTypeConflicts(occurance: Occurance): Conflict { - if(_.includes(_.keys(occurance.dataType), "null") && _.size(occurance.dataType) === 1) { + if (_.includes(_.keys(occurance.dataType), "null") && _.size(occurance.dataType) === 1) { return { type: constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD, // Should be used only to return the name of field instead of path // property: Object.keys(occurance.property)[0], property: _.replace(Object.keys(occurance.path)[0], "$.", ""), conflicts: occurance.dataType, - resolution: { "value": occurance.dataType, "type": constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD }, + resolution: { "value": occurance.dataType, "type": constants.SCHEMA_RESOLUTION_TYPE.NULL_FIELD }, values: _.keys(occurance.dataType), severity: constants.SEVERITY["MUST-FIX"], path: _.replace(Object.keys(occurance.absolutePath)[0], "$.", ""), @@ -158,26 +158,26 @@ export class SchemaAnalyser { * Method to iterate over the schema object in a recursive and flatten the required properties */ public flattenSchema(sample: Map): FlattenSchema[] { - let array = new Array(); + const array: any[] = []; const recursive = (data: any, path: string, requiredProps: string[], schemaPath: string) => { _.map(data, (value, key) => { - let isMultipleTypes = ''; - if(_.has(value, 'anyOf')) isMultipleTypes = 'anyOf'; - if(_.has(value, 'oneOf')) isMultipleTypes = 'oneOf'; - if (_.isPlainObject(value) && (_.has(value, 'properties'))) { - array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) - recursive(value['properties'], `${path}.${key}`, value['required'], `${schemaPath}.properties.${key}`); - } else if(_.isPlainObject(value)) { - if (value.type === 'array') { - array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) - if (_.has(value, 'items') && _.has(value["items"], 'properties')) { - recursive(value["items"]['properties'], `${path}.${key}[*]`, value["items"]['required'], `${schemaPath}.properties.${key}.items`); + let isMultipleTypes = ""; + if (_.has(value, "anyOf")) isMultipleTypes = "anyOf"; + if (_.has(value, "oneOf")) isMultipleTypes = "oneOf"; + if (_.isPlainObject(value) && (_.has(value, "properties"))) { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + recursive(value["properties"], `${path}.${key}`, value["required"], `${schemaPath}.properties.${key}`); + } else if (_.isPlainObject(value)) { + if (value.type === "array") { + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + if (_.has(value, "items") && _.has(value["items"], "properties")) { + recursive(value["items"]["properties"], `${path}.${key}[*]`, value["items"]["required"], `${schemaPath}.properties.${key}.items`); } - } else if(isMultipleTypes != '') { - array.push(this._flattenSchema(key, value[isMultipleTypes][0].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) - array.push(this._flattenSchema(key, value[isMultipleTypes][1].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) + } else if (isMultipleTypes != "") { + array.push(this._flattenSchema(key, value[isMultipleTypes][0].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) + array.push(this._flattenSchema(key, value[isMultipleTypes][1].type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) } else { - array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value['format'])) + array.push(this._flattenSchema(key, value.type, _.includes(requiredProps, key), `${path}.${key}`, `${schemaPath}.properties.${key}`, value["format"])) } } }) diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts index cc52309e..4bd19c91 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts @@ -4,7 +4,7 @@ export class SchemaArrayValidator { public validate(schemas: any) { _.map(schemas, (schema: any, index: number) => { Object.entries(schema).map(([schemaKey, schemaValue]) => { - if (typeof schemaValue === 'object') { + if (typeof schemaValue === "object") { this.handleNestedObject(index, `${schemaKey}`, schemaValue, schemas); } }); @@ -13,15 +13,15 @@ export class SchemaArrayValidator { } private checkForInvalidArray(value: any) { - if (_.has(value, 'items') && _.has(value, 'properties')) - _.unset(value, 'properties'); + if (_.has(value, "items") && _.has(value, "properties")) + _.unset(value, "properties"); } private handleNestedObject(index: any, path: string, value: any, schemas: any) { Object.entries(value).map(([nestedKey, nestedValue]: any) => { - if (typeof nestedValue === 'object') { + if (typeof nestedValue === "object") { this.handleNestedObject(index, `${path}.${nestedKey}`, nestedValue, schemas) - } else if (nestedValue.type === 'array' && (nestedValue.items != false)) { + } else if (nestedValue.type === "array" && (nestedValue.items != false)) { this.checkForInvalidArray(nestedValue); let isValidArray = true; if(_.isEqual(_.get(schemas[0], `${path}.${nestedKey}.type`), _.get(schemas[index], `${path}.${nestedKey}.type`))) { @@ -33,14 +33,14 @@ export class SchemaArrayValidator { if (!isValidArray) { this.deleteItemsAndSetAdditionalProperties(schemas, `${path}.${nestedKey}`) } - } else if (nestedValue.type === 'array' && (nestedValue.items == false)) { + } else if (nestedValue.type === "array" && (nestedValue.items == false)) { this.deleteItemsAndSetAdditionalProperties(schemas, `${path}.${nestedKey}`) } }) } private deleteItemsAndSetAdditionalProperties(schemas: any, path: string) { - _.map((schemas), (schema: any, index: number) => { + _.map((schemas), (schema: any) => { if (!isUndefined(_.get(schema, path))) { _.unset(schema, `${path}`) _.set(schema, `${path}.type`, "array"); diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts index 832ce7a5..99f15ea2 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts @@ -6,21 +6,21 @@ import { UniqueValues, FieldSchema, RollupSummary } from "../../types/SchemaMode export const generateRollupSummary = (uniqueValues: UniqueValues) => { const summary: RollupSummary = {}; Object.entries(uniqueValues).map(([field, value]) => { - let data: Record = {}; + const data: Record = {}; _.map(value, (item: string) => { if (!_.has(data, [field, item])) _.set(data, [field, item], 1); else data[field][item] += 1; }); const resultData: Record = {}; _.map(_.keys(data), (path: string) => { - Object.entries(data[path]).map(([key, value]: any) => { + Object.entries(data[path]).map(([, value]: any) => { const totalValue = _.sum(_.values(data[path])); const ratio = Math.round((value / totalValue) * 100); if (!_.has(resultData, path)) _.set(resultData, path, ratio); else if (ratio > _.get(resultData, path)) _.set(resultData, path, ratio); }); - let fieldName = parseSchemaPath(path); + const fieldName = parseSchemaPath(path); summary[fieldName] = { path: `$.${path}`, cardinality: 100 - _.get(resultData, path), diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts index ad3bb713..b71de416 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts @@ -13,7 +13,7 @@ export const dataMappingPaths = { "number": "number.store_format.number.jsonSchema", "object": "object.store_format.object.jsonSchema", "array": "array.store_format.array.jsonSchema", -} +} export class SchemaHandler { private typeToMethod = { @@ -52,23 +52,23 @@ export class SchemaHandler { private setNulltype(schema: any, conflict: ConflictTypes): any { const { absolutePath, schema: { resolution: { value } } } = conflict; const dataTypes: any = []; - _.forEach(DataMappings, (valueItem, keyItem) => { - _.forEach(_.get(valueItem, 'store_format'), (subValue, subKey) => { + _.forEach(DataMappings, (valueItem) => { + _.forEach(_.get(valueItem, "store_format"), (subValue) => { if (!_.find(dataTypes, ["type", subValue["jsonSchema"]])) dataTypes.push({ type: subValue["jsonSchema"] }) }) }); const arrivalDataTypes: any = _.keys(DataMappings).map((key: any) => ({ type: key })); - _.set(schema, `${absolutePath}.type`, 'null'); + _.set(schema, `${absolutePath}.type`, "null"); _.set(schema, `${absolutePath}.arrivalOneOf`, arrivalDataTypes); return _.set(schema, `${absolutePath}.oneof`, dataTypes); } private updateRequiredProp(schema: any, value: ConflictTypes): any { - const absolutePath = value.absolutePath.replace(value.required.property, value.required.property.replace('.', '$')) - const subStringArray: string[] = _.split(absolutePath, '.'); - const subString: string = _.join(_.slice(subStringArray, 0, subStringArray.length - 2), '.'); + const absolutePath = value.absolutePath.replace(value.required.property, value.required.property.replace(".", "$")) + const subStringArray: string[] = _.split(absolutePath, "."); + const subString: string = _.join(_.slice(subStringArray, 0, subStringArray.length - 2), "."); const path: string = _.isEmpty(subString) ? "required" : `${subString}.required` const requiredList: string[] = _.get(schema, path) const newProperty: string = value.required.property @@ -77,7 +77,7 @@ export class SchemaHandler { } private getArrivalSuggestions(schema: any, fieldData: any, property: any, type: string) { - let arrivalSuggestions: any = []; + const arrivalSuggestions: any = []; const types = _.get(fieldData, type); types && types.map((item: any) => { const storeFormat = _.get(dataMappingPaths, item.type); @@ -86,7 +86,7 @@ export class SchemaHandler { if (arrivalSuggestions.length > 0) _.set(schema, `${property}.arrivalOneOf`, arrivalSuggestions); return; - }; + } private getArrivalFormat(schema: any, fieldData: any, property: any, type: string) { const types = _.get(fieldData, type); @@ -105,11 +105,11 @@ export class SchemaHandler { const arrivalConflictExists = _.filter(suggestions, (suggestion) => _.has(suggestion, "arrivalConflict")); switch (true) { // Add arrival conflicts if there is arrival conflict in suggestions - case _.has(fieldData, 'oneof') && arrivalConflictExists.length > 0: - return this.getArrivalSuggestions(schema, fieldData, property, 'oneof') + case _.has(fieldData, "oneof") && arrivalConflictExists.length > 0: + return this.getArrivalSuggestions(schema, fieldData, property, "oneof") // Add arrival type if there are no arrival type conflicts case arrivalConflictExists.length === 0: - return this.getArrivalFormat(schema, fieldData, property, 'oneof') + return this.getArrivalFormat(schema, fieldData, property, "oneof") default: break; } @@ -126,22 +126,22 @@ export class SchemaHandler { } private checkForInvalidArray(value: any) { - if (_.has(value, 'items') && _.has(value, 'properties')) - _.unset(value, 'properties'); + if (_.has(value, "items") && _.has(value, "properties")) + _.unset(value, "properties"); } private updateMappings(schema: Map) { const recursive = (data: any) => { - _.map(data, (value, key) => { + _.map(data, (value) => { if (_.isPlainObject(value)) { - if ((_.has(value, 'properties'))) { - recursive(value['properties']); + if ((_.has(value, "properties"))) { + recursive(value["properties"]); } - if (value.type === 'array') { - if (_.has(value, 'items') && _.has(value["items"], 'properties')) { - recursive(value["items"]['properties']); + if (value.type === "array") { + if (_.has(value, "items") && _.has(value["items"], "properties")) { + recursive(value["items"]["properties"]); } - if (_.has(value, 'items') && _.has(value, 'properties')) + if (_.has(value, "items") && _.has(value, "properties")) this.checkForInvalidArray(value); this.updateStoreType(value, _.get(value, "type")); } else { diff --git a/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts b/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts index bb87fe32..fcaba4a5 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts @@ -6,7 +6,7 @@ import { SchemaSuggestionTemplate } from "./Template" export class SuggestionTemplate { public createSuggestionTemplate(sample: ConflictTypes[]): SuggestionsTemplate[] { - return _.map(sample, (value, key) => { + return _.map(sample, (value) => { const dataTypeSuggestions = this.getSchemaMessageTemplate(value.schema) const requiredSuggestions = this.getRequiredMessageTemplate(value.required) const formatSuggestions = this.getPropertyFormatTemplate(value.formats) @@ -24,7 +24,7 @@ export class SuggestionTemplate { message = SchemaSuggestionTemplate.getSchemaNullTypeMessage(object.conflicts, object.property); advice = SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.NULL_TYPE_PROPERTY.ADVICE; } else { - let { conflictMessage, arrivalFormatMessage } = SchemaSuggestionTemplate.getSchemaDataTypeMessage(object.conflicts, object.property); + const { conflictMessage, arrivalFormatMessage } = SchemaSuggestionTemplate.getSchemaDataTypeMessage(object.conflicts, object.property); message = conflictMessage; arrival_format_message = arrivalFormatMessage; advice = SchemaSuggestionTemplate.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.ADVICE; diff --git a/api-service/src/v2/services/SchemaGenerateService/Template.ts b/api-service/src/v2/services/SchemaGenerateService/Template.ts index 49c86228..9beb3a46 100644 --- a/api-service/src/v2/services/SchemaGenerateService/Template.ts +++ b/api-service/src/v2/services/SchemaGenerateService/Template.ts @@ -71,7 +71,7 @@ export const SchemaSuggestionTemplate = { updatedConflicts[types[0]] = value; } }); - let response: Record = { + const response: Record = { conflictMessage: _.template( `${this.TEMPLATES.SCHEMA_SUGGESTION.CREATE.DATATYPE_PROPERTY.MESSAGE} at property: '${property}'. The property type <% _.map(conflicts, (value, key, list) => { %><%= key %>: <%= value %> time(s)<%= _.last(list) === value ? '' : ', ' %><% }); %><%= _.isEmpty(conflicts) ? '' : '' %>`)({ conflicts }), arrivalFormatMessage: null, diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts index 838538b7..dc4afbfc 100644 --- a/api-service/src/v2/services/TableGenerator.ts +++ b/api-service/src/v2/services/TableGenerator.ts @@ -12,31 +12,31 @@ class BaseTableGenerator { */ flattenSchema = (dataSchema: Record, type: string) : Record[] => { - let properties: Record[] = [] + const properties: Record[] = [] const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { _.mapKeys(schema, function(value, parentKey) { - const newKey = (prev) ? _.join([prev, parentKey], '.') : parentKey; - const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], '') : _.join(["$.['", parentKey, "']"], ''); - switch(value['type']) { - case 'object': - flatten(_.get(value, 'properties'), newKey, newExpr); + const newKey = (prev) ? _.join([prev, parentKey], ".") : parentKey; + const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], "") : _.join(["$.['", parentKey, "']"], ""); + switch(value["type"]) { + case "object": + flatten(_.get(value, "properties"), newKey, newExpr); break; - case 'array': - if(type === "druid" && _.get(value, 'items.type') == 'object' && _.get(value, 'items.properties')) { - _.mapKeys(_.get(value, 'items.properties'), function(value, childKey) { - const objChildKey = _.join([newKey, childKey], '.') - properties.push(_.merge(_.pick(value, ['type', 'arrival_format', 'is_deleted']), {expr: _.join([newExpr,"[*].['",childKey,"']"], ''), name: objChildKey, data_type: 'array'})) + case "array": + if(type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { + _.mapKeys(_.get(value, "items.properties"), function(value, childKey) { + const objChildKey = _.join([newKey, childKey], ".") + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), {expr: _.join([newExpr,"[*].['",childKey,"']"], ""), name: objChildKey, data_type: "array"})) }) } else { - properties.push(_.merge(_.pick(value, ['arrival_format', 'data_type', 'is_deleted']), {expr: newExpr+'[*]', name: newKey, type: _.get(value, 'items.type')})) + properties.push(_.merge(_.pick(value, ["arrival_format", "data_type", "is_deleted"]), {expr: newExpr+"[*]", name: newKey, type: _.get(value, "items.type")})) } break; default: - properties.push(_.merge(_.pick(value, ['type', 'arrival_format', 'data_type', 'is_deleted']), {expr: newExpr, name: newKey})) + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "data_type", "is_deleted"]), {expr: newExpr, name: newKey})) } }); } - flatten(_.get(dataSchema, 'properties'), undefined, undefined) + flatten(_.get(dataSchema, "properties"), undefined, undefined) return properties } @@ -58,8 +58,8 @@ class BaseTableGenerator { const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); const properties = instance.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { - _.set(prop, 'name', _.join([denormField.denorm_out_field, prop.name], '.')); - _.set(prop, 'expr', _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); + _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], ".")); + _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); return prop; }); dataFields.push(...transformProps); @@ -182,12 +182,12 @@ class TableGenerator extends BaseTableGenerator { getHudiIngestionSpecForUpdate = (dataset: Record, existingHudiSpec: Record, allFields: Record[], datasourceRef: string) => { - let newHudiSpec = this.getHudiIngestionSpecForCreate(dataset, allFields, datasourceRef) + const newHudiSpec = this.getHudiIngestionSpecForCreate(dataset, allFields, datasourceRef) const newColumnSpec = newHudiSpec.schema.columnSpec; - let oldColumnSpec = existingHudiSpec.schema.columnSpec; - let currIndex = _.get(_.maxBy(oldColumnSpec, 'index'), 'index') as unknown as number - const newColumns = _.differenceBy(newColumnSpec, oldColumnSpec, 'name'); + const oldColumnSpec = existingHudiSpec.schema.columnSpec; + let currIndex = _.get(_.maxBy(oldColumnSpec, "index"), "index") as unknown as number + const newColumns = _.differenceBy(newColumnSpec, oldColumnSpec, "name"); if(_.size(newColumns) > 0) { _.each(newColumns, (col) => { oldColumnSpec.push({ @@ -197,7 +197,7 @@ class TableGenerator extends BaseTableGenerator { }) }) } - _.set(newHudiSpec, 'schema.columnSpec', oldColumnSpec) + _.set(newHudiSpec, "schema.columnSpec", oldColumnSpec) return newHudiSpec; } @@ -227,10 +227,10 @@ class TableGenerator extends BaseTableGenerator { } private getHudiColumnType = (field: Record) : string => { - if(field.data_type === 'array' && field.arrival_format !== 'array') { + if(field.data_type === "array" && field.arrival_format !== "array") { return "array"; } - if(field.data_type === 'array' && field.arrival_format === 'array') { + if(field.data_type === "array" && field.arrival_format === "array") { switch(field.type) { case "string": return "array" diff --git a/api-service/src/v2/types/ConfigModels.ts b/api-service/src/v2/types/ConfigModels.ts index 320cbb32..a03ec5f4 100644 --- a/api-service/src/v2/types/ConfigModels.ts +++ b/api-service/src/v2/types/ConfigModels.ts @@ -1,5 +1,5 @@ import { IngestionConfig } from "./IngestionModels"; -import { IDataSourceRules, IRules } from "./QueryModels"; +import { IDataSourceRules } from "./QueryModels"; export interface ExtractionConfig { is_batch_event: boolean; From 0e1b392f8e2924fbaa9017bedfcdf6230bf1f267 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sun, 21 Jul 2024 22:14:21 +0530 Subject: [PATCH 042/235] #OBS-I116: lint fixes --- .../GenerateDataSchema/GenerateDataSchema.ts | 4 +- api-service/src/v2/middlewares/errors.ts | 2 +- .../SchemaGenerateService/ConfigSuggester.ts | 4 +- .../DataSchemaService.ts | 2 +- .../SchemaGenerateService/SchemaAnalyser.ts | 12 +-- .../SchemaGenerateService/SchemaHandler.ts | 6 +- api-service/src/v2/services/TableGenerator.ts | 81 +++++++++---------- 7 files changed, 55 insertions(+), 56 deletions(-) diff --git a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts index 3d16f5b2..a37e5d13 100644 --- a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts +++ b/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts @@ -80,7 +80,7 @@ const checkJsonSchema = (sample: Map): boolean => { const removeNonIndexColumns = (schema: any) => { if (schema.properties) { - Object.entries(schema.properties).map(([key, property]: any) => { + Object.entries(schema.properties).map(([, property]: any) => { _.unset(schema, "required"); removeNonIndexColumns(property) }); @@ -95,7 +95,7 @@ const removeNonIndexColumns = (schema: any) => { const removeFormats = (schema: any) => { if (schema.properties) { - Object.entries(schema.properties).map(([key, property]: any) => { + Object.entries(schema.properties).map(([, property]: any) => { // Removing format to avoid schema validation issues const isDateTypeField = ["date-time", "date", "epoch"].includes((property as any).format); if (isDateTypeField && _.get(property, "data_type") === "string") { diff --git a/api-service/src/v2/middlewares/errors.ts b/api-service/src/v2/middlewares/errors.ts index 2ca96caf..b15d561a 100644 --- a/api-service/src/v2/middlewares/errors.ts +++ b/api-service/src/v2/middlewares/errors.ts @@ -1,4 +1,4 @@ -import { NextFunction, Request, Response } from "express"; +import { Request, Response } from "express"; import logger from "../logger"; import { ResponseHandler } from "../helpers/ResponseHandler"; import _ from "lodash"; diff --git a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts index 0eb13bbb..8834129d 100644 --- a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts +++ b/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts @@ -31,12 +31,12 @@ export class ConfigSuggestor { private analyzeConflicts(conflicts: ConflictTypes[]): DataSetConfig { const typeFormatsConflict: ConflictTypes[] = _.filter(conflicts, (o) => !_.isEmpty(o.formats)); - const ingestionConfig: IngestionConfig = this.ingestionConfig(typeFormatsConflict) + const ingestionConfig: IngestionConfig = this.ingestionConfig() const processingConfig: DatasetProcessing = this.processingConfig(typeFormatsConflict) return { "indexConfiguration": ingestionConfig, "processing": processingConfig } } - private ingestionConfig(conflicts: ConflictTypes[]): any { + private ingestionConfig(): any { return { "index": Object.assign(ingestionConfig.indexCol), "rollupSuggestions": this.rollupInfo }; } diff --git a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts index 404ad24b..67bde9e7 100644 --- a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts +++ b/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts @@ -76,9 +76,9 @@ export class SchemaInference { isValidTimestamp(value: any) { const dataType = typeof value; + const epochRegex = /^\d+$/ig; switch (dataType) { case "string": - const epochRegex = /^\d+$/ig; if(epochRegex.test(value)){ const parsedValue = parseInt(value, 10); // Timestamp should be greater than Jan 01 2000 00:00:00 UTC/GMT in seconds diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts index 4f0cb3ce..a3a24b1d 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts @@ -25,8 +25,8 @@ export class SchemaAnalyser { const result: FlattenSchema[] = _.flatten(this.schemas.map(element => { return this.flattenSchema(new Map(Object.entries(element))); })) - const conflicts = Object.entries(_.groupBy(result, "path")).map(([key, value]) => { - return this.getSchemaConflictTypes(this.getOccurance(value, key)) + const conflicts = Object.entries(_.groupBy(result, "path")).map(([, value]) => { + return this.getSchemaConflictTypes(this.getOccurance(value)) }) return _.filter(conflicts, obj => (!_.isEmpty(obj.schema) || !_.isEmpty(obj.required) || !_.isEmpty(obj.formats))) } @@ -123,7 +123,7 @@ export class SchemaAnalyser { */ private findOptionalPropConflicts(occurance: Occurance): Conflict { const maxOccurance: number = 1 - const requiredCount = _.map(occurance.property, (value, key) => { + const requiredCount = _.map(occurance.property, (value) => { return value })[0] @@ -148,9 +148,9 @@ export class SchemaAnalyser { * * Method to get the occurance of the given key from the given object */ - private getOccurance(arrayOfObjects: object[], key: string): Occurance { - const result = _(arrayOfObjects).flatMap(obj => _.toPairs(obj)).groupBy(([key, value]) => key) - .mapValues(group => _.countBy(group, ([key, value]) => value)).value(); + private getOccurance(arrayOfObjects: object[]): Occurance { + const result = _(arrayOfObjects).flatMap(obj => _.toPairs(obj)).groupBy(([key]) => key) + .mapValues(group => _.countBy(group, ([, value]) => value)).value(); return { property: result.property, dataType: result.dataType, isRequired: result.isRequired, path: result.path, absolutePath: result.absolutePath, format: result.formate }; } diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts index b71de416..45d76c9c 100644 --- a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts +++ b/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts @@ -39,18 +39,18 @@ export class SchemaHandler { } private updateDataTypes(schema: any, conflict: ConflictTypes): any { - const { absolutePath, schema: { resolution: { value } } } = conflict; + const { absolutePath, schema: { resolution } } = conflict; return _.set(schema, `${absolutePath}`, { ...schema[absolutePath], ...{ - type: conflict.schema.resolution["value"], + type: resolution.value, oneof: conflict.schema.values.map(key => ({ type: key })), } }); } private setNulltype(schema: any, conflict: ConflictTypes): any { - const { absolutePath, schema: { resolution: { value } } } = conflict; + const { absolutePath } = conflict; const dataTypes: any = []; _.forEach(DataMappings, (valueItem) => { _.forEach(_.get(valueItem, "store_format"), (subValue) => { diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/v2/services/TableGenerator.ts index dc4afbfc..eb3065ae 100644 --- a/api-service/src/v2/services/TableGenerator.ts +++ b/api-service/src/v2/services/TableGenerator.ts @@ -10,29 +10,29 @@ class BaseTableGenerator { * @param dataSchema * @returns properties Record[] */ - flattenSchema = (dataSchema: Record, type: string) : Record[] => { + flattenSchema = (dataSchema: Record, type: string): Record[] => { const properties: Record[] = [] const flatten = (schema: Record, prev: string | undefined, prevExpr: string | undefined) => { - _.mapKeys(schema, function(value, parentKey) { + _.mapKeys(schema, function (value, parentKey) { const newKey = (prev) ? _.join([prev, parentKey], ".") : parentKey; const newExpr = (prevExpr) ? _.join([prevExpr, ".['", parentKey, "']"], "") : _.join(["$.['", parentKey, "']"], ""); - switch(value["type"]) { - case "object": + switch (value["type"]) { + case "object": flatten(_.get(value, "properties"), newKey, newExpr); break; case "array": - if(type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { - _.mapKeys(_.get(value, "items.properties"), function(value, childKey) { + if (type === "druid" && _.get(value, "items.type") == "object" && _.get(value, "items.properties")) { + _.mapKeys(_.get(value, "items.properties"), function (value, childKey) { const objChildKey = _.join([newKey, childKey], ".") - properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), {expr: _.join([newExpr,"[*].['",childKey,"']"], ""), name: objChildKey, data_type: "array"})) + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "is_deleted"]), { expr: _.join([newExpr, "[*].['", childKey, "']"], ""), name: objChildKey, data_type: "array" })) }) } else { - properties.push(_.merge(_.pick(value, ["arrival_format", "data_type", "is_deleted"]), {expr: newExpr+"[*]", name: newKey, type: _.get(value, "items.type")})) + properties.push(_.merge(_.pick(value, ["arrival_format", "data_type", "is_deleted"]), { expr: newExpr + "[*]", name: newKey, type: _.get(value, "items.type") })) } break; default: - properties.push(_.merge(_.pick(value, ["type", "arrival_format", "data_type", "is_deleted"]), {expr: newExpr, name: newKey})) + properties.push(_.merge(_.pick(value, ["type", "arrival_format", "data_type", "is_deleted"]), { expr: newExpr, name: newKey })) } }); } @@ -51,12 +51,11 @@ class BaseTableGenerator { getAllFields = async (dataset: Record, type: string): Promise[]> => { const { data_schema, denorm_config, transformations_config } = dataset - const instance = this; - let dataFields = instance.flattenSchema(data_schema, type); + let dataFields = this.flattenSchema(data_schema, type); if (!_.isEmpty(denorm_config.denorm_fields)) { for (const denormField of denorm_config.denorm_fields) { const denormDataset: any = await datasetService.getDataset(denormField.dataset_id, ["data_schema"], true); - const properties = instance.flattenSchema(denormDataset.data_schema, type); + const properties = this.flattenSchema(denormDataset.data_schema, type); const transformProps = _.map(properties, (prop) => { _.set(prop, "name", _.join([denormField.denorm_out_field, prop.name], ".")); _.set(prop, "expr", _.replace(prop.expr, "$", "$." + denormField.denorm_out_field)); @@ -85,7 +84,7 @@ class BaseTableGenerator { class TableGenerator extends BaseTableGenerator { getDruidIngestionSpec = (dataset: Record, allFields: Record[], datasourceRef: string) => { - + const { dataset_config, router_config } = dataset return { "type": "kafka", @@ -109,22 +108,21 @@ class TableGenerator extends BaseTableGenerator { } } } - + private getDruidDimensions = (allFields: Record[], timestampKey: string, partitionKey: string | undefined) => { const dataFields = _.cloneDeep(allFields); - if(partitionKey) { // Move the partition column to the top of the dimensions - const partitionCol = _.remove(dataFields, {name: partitionKey}) - if(partitionCol && _.size(partitionCol) > 0) { + if (partitionKey) { // Move the partition column to the top of the dimensions + const partitionCol = _.remove(dataFields, { name: partitionKey }) + if (partitionCol && _.size(partitionCol) > 0) { dataFields.unshift(partitionCol[0]) } } - _.remove(dataFields, {name: timestampKey}) - const instance = this; + _.remove(dataFields, { name: timestampKey }) return _.union( _.map(dataFields, (field) => { return { - "type": instance.getDruidDimensionType(field.data_type), + "type": this.getDruidDimensionType(field.data_type), "name": field.name } }), @@ -132,7 +130,7 @@ class TableGenerator extends BaseTableGenerator { ) } - private getDruidDimensionType = (data_type: string):string => { + private getDruidDimensionType = (data_type: string): string => { switch (data_type) { case "number": return "double"; case "integer": return "long"; @@ -188,7 +186,7 @@ class TableGenerator extends BaseTableGenerator { const oldColumnSpec = existingHudiSpec.schema.columnSpec; let currIndex = _.get(_.maxBy(oldColumnSpec, "index"), "index") as unknown as number const newColumns = _.differenceBy(newColumnSpec, oldColumnSpec, "name"); - if(_.size(newColumns) > 0) { + if (_.size(newColumns) > 0) { _.each(newColumns, (col) => { oldColumnSpec.push({ "type": col.type, @@ -201,17 +199,16 @@ class TableGenerator extends BaseTableGenerator { return newHudiSpec; } - private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string) : Record[] => { + private getHudiColumnSpec = (allFields: Record[], primaryKey: string, partitionKey: string, timestampKey: string): Record[] => { - const instance = this; const dataFields = _.cloneDeep(allFields); - _.remove(dataFields, {name: primaryKey}) - _.remove(dataFields, {name: partitionKey}) - _.remove(dataFields, {name: timestampKey}) + _.remove(dataFields, { name: primaryKey }) + _.remove(dataFields, { name: partitionKey }) + _.remove(dataFields, { name: timestampKey }) let index = 1; - const transformFields = _.map(dataFields, (field) => { + const transformFields = _.map(dataFields, (field) => { return { - "type": instance.getHudiColumnType(field), + "type": this.getHudiColumnType(field), "name": field.name, "index": index++ } @@ -226,12 +223,12 @@ class TableGenerator extends BaseTableGenerator { return transformFields; } - private getHudiColumnType = (field: Record) : string => { - if(field.data_type === "array" && field.arrival_format !== "array") { + private getHudiColumnType = (field: Record): string => { + if (field.data_type === "array" && field.arrival_format !== "array") { return "array"; } - if(field.data_type === "array" && field.arrival_format === "array") { - switch(field.type) { + if (field.data_type === "array" && field.arrival_format === "array") { + switch (field.type) { case "string": return "array" case "number": @@ -244,11 +241,11 @@ class TableGenerator extends BaseTableGenerator { return "array" } } - switch(field.arrival_format) { + switch (field.arrival_format) { case "text": return "string" case "number": - switch(field.data_type) { + switch (field.data_type) { case "integer": return "int" case "epoch": @@ -260,7 +257,7 @@ class TableGenerator extends BaseTableGenerator { case "long": return "long" default: - return "double" + return "double" } case "integer": return "int" @@ -271,13 +268,15 @@ class TableGenerator extends BaseTableGenerator { } } - private getHudiFields = (allFields: Record[]) : Record[] => { + private getHudiFields = (allFields: Record[]): Record[] => { + const regexString = "[\\[\\]'\\*]"; + const regex = new RegExp(regexString, "g"); return _.union( _.map(allFields, (field) => { return { type: "path", - expr: _.replace(field.expr, /[\[\]'\*]/g, ""), + expr: _.replace(field.expr, regex, ""), name: field.name } }), @@ -285,15 +284,15 @@ class TableGenerator extends BaseTableGenerator { ) } - private getPrimaryKey = (dataset: Record) : string => { + private getPrimaryKey = (dataset: Record): string => { return dataset.dataset_config.keys_config.data_key; } - private getHudiPartitionKey = (dataset: Record) : string => { + private getHudiPartitionKey = (dataset: Record): string => { return dataset.dataset_config.keys_config.partition_key || dataset.dataset_config.keys_config.timestamp_key; } - private getTimestampKey = (dataset: Record) : string => { + private getTimestampKey = (dataset: Record): string => { return dataset.dataset_config.keys_config.timestamp_key; } } From d206d87fa07c07f3abc919e2e83e8d9a676e7c38 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sun, 21 Jul 2024 22:43:55 +0530 Subject: [PATCH 043/235] #OBS-I116: Dataset status transition test cases --- api-service/src/v2/middlewares/errors.ts | 6 +++--- .../DatasetStatusTransition/DatasetDelete.spec.ts | 9 ++++++++- .../DatasetStatusTransition/DatasetLive.spec.ts | 2 +- .../DatasetReadyToPublish.spec.ts | 2 +- .../DatasetStatusTransition/DatasetRetire.spec.ts | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/api-service/src/v2/middlewares/errors.ts b/api-service/src/v2/middlewares/errors.ts index b15d561a..96c8069e 100644 --- a/api-service/src/v2/middlewares/errors.ts +++ b/api-service/src/v2/middlewares/errors.ts @@ -1,10 +1,10 @@ -import { Request, Response } from "express"; +import { NextFunction, Request, Response } from "express"; import logger from "../logger"; import { ResponseHandler } from "../helpers/ResponseHandler"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; -export const errorHandler = (err: Error, req: Request, res: Response) => { +export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => { logger.error({ path: req.url, req: req.body , ...err }) const errorMessage = {name: err.name, message: err.message}; @@ -12,7 +12,7 @@ export const errorHandler = (err: Error, req: Request, res: Response) => { }; -export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response) => { +export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, next: NextFunction) => { logger.error({ path: req.url, req: req.body, resmsgid: _.get(res, "resmsgid") , ...obsrvErr }) ResponseHandler.obsrvErrorResponse(obsrvErr, req, res); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts index 29a8584d..f3de6b4f 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts @@ -10,6 +10,7 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; import { DatasetSourceConfigDraft } from "../../../models/DatasetSourceConfigDraft"; import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); @@ -40,6 +41,12 @@ describe("DATASET STATUS TRANSITION DELETE", () => { chai.spy.on(DatasetDraft, "destroy", () => { return Promise.resolve({}) }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) chai .request(app) .post("/v2/datasets/status-transition") @@ -71,7 +78,7 @@ describe("DATASET STATUS TRANSITION DELETE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to delete") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 4623a0e3..d7dbfb36 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -59,7 +59,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to perform status transition to live") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }) diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index 83173c68..b8f83879 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -59,7 +59,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to perform status transition to ready to publish") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index 21d721d3..021b7305 100644 --- a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -166,7 +166,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found to retire") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }) From da4928c960c0e3f1a2c1fdddda8ec7a9d52f5dd2 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 22 Jul 2024 13:11:41 +0530 Subject: [PATCH 044/235] #OBS-I126 : added multiple requests example --- .../updated_v2_collection.json | 215 ++- .../swagger-doc/v2_updated_doc_openapi.yml | 1239 ++++++++++++++--- 2 files changed, 1283 insertions(+), 171 deletions(-) diff --git a/api-service/postman-collection/updated_v2_collection.json b/api-service/postman-collection/updated_v2_collection.json index 527881cd..45dc3ac3 100644 --- a/api-service/postman-collection/updated_v2_collection.json +++ b/api-service/postman-collection/updated_v2_collection.json @@ -1,10 +1,10 @@ { "info": { - "_postman_id": "04207e75-91d2-4896-a6ef-ea6a12c4018f", + "_postman_id": "99bdb8d0-6d3a-4ba2-9044-ec5263e54c95", "name": "V2 docs", "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", "_exporter_id": "33916975", - "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/33916975-04207e75-91d2-4896-a6ef-ea6a12c4018f?action=share&source=collection_link&creator=33916975" + "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/33916975-99bdb8d0-6d3a-4ba2-9044-ec5263e54c95?action=share&source=collection_link&creator=33916975" }, "item": [ { @@ -2042,6 +2042,217 @@ "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:02:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"add9dbe0-f362-4f99-890c-3387c998a049\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/required must have required property 'msgid'\"\n }\n}" } ] + }, + { + "name": "Data schema generator", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ],\n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "response": [ + { + "name": "Data schema generated successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ],\n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "10760" + }, + { + "key": "ETag", + "value": "W/\"2a08-QF5x1q0kIlfE9XU/pa9IboJuY8I\"" + }, + { + "key": "Date", + "value": "Mon, 22 Jul 2024 07:02:50 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:32:50+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1309aea0-9a97-46e9-bc5e-a16a8a7fb624\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'sid'. The property sid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.sid\"\n },\n {\n \"message\": \"The Property 'context.sid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.sid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.cdata[*].id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.cdata.items.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'l1'. The property l1: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.rollup.properties.l1\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uid'. The property uid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.uid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"object\": {\n \"type\": \"object\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'object'. The property object: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.object\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'tags'. The property tags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.tags\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'pageid'. The property pageid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.pageid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'subtype'. The property subtype: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.subtype\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uri'. The property uri: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.uri\"\n },\n {\n \"message\": \"The Property 'edata.uri' appears to be 'uri' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.uri\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"visits\": {\n \"type\": \"array\",\n \"items\": false,\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'visits'. The property visits: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.visits\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"level\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"message\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"params\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'syncts'. The property syncts: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.syncts\"\n },\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: '@timestamp'. The property @timestamp: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.@timestamp\"\n },\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'ex_processed'. The property ex_processed: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags.properties.ex_processed\"\n }\n ],\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'flags'. The property flags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"mid\": {\n \"path\": \"$.mid\",\n \"cardinality\": 67,\n \"index\": false\n },\n \"actor.id\": {\n \"path\": \"$.actor.properties.id\",\n \"cardinality\": 56,\n \"index\": false\n },\n \"context.sid\": {\n \"path\": \"$.context.properties.sid\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"edata.uri\": {\n \"path\": \"$.edata.properties.uri\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"context.cdata[*].id\": {\n \"path\": \"$.context.properties.cdata.items.properties.id\",\n \"cardinality\": 62,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"mid\",\n \"context.cdata[*].id\",\n \"actor.id\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + }, + { + "name": "Failure: Invalid request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "364" + }, + { + "key": "ETag", + "value": "W/\"16c-tfKVtCWTjNkWCtH8cFw1RrzbgV0\"" + }, + { + "key": "Date", + "value": "Mon, 22 Jul 2024 07:03:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:33:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bbcc86c2-042d-4f77-bb6e-e1c9116df570\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'data'\"\n }\n}" + }, + { + "name": "Failure: Invalid request (config not provided)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ]\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "366" + }, + { + "key": "ETag", + "value": "W/\"16e-YeX++2sGUWsjHqjP2GoTgqujU+c\"" + }, + { + "key": "Date", + "value": "Mon, 22 Jul 2024 07:05:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:35:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1f856c5e-37f0-41e9-96fb-642471228da2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'config'\"\n }\n}" + } + ] } ] } diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml index 87625317..fef21941 100644 --- a/api-service/swagger-doc/v2_updated_doc_openapi.yml +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -16,61 +16,74 @@ paths: requestBody: content: application/json: - schema: - type: object - example: - id: api.datasets.create - ver: v1 - ts: '2024-04-10T16:10:50+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - request: - type: event - name: sb-telemetry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: true - extraction_key: events - dedup_config: - drop_duplicates: true - dedup_key: id - dedup_config: - drop_duplicates: true - dedup_key: mid - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - mid: - type: string - arrival_format: text - data_type: string - ets: - type: integer - arrival_format: number - data_type: epoch - eid: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - denorm_fields: - - denorm_key: eid - denorm_out_field: userdata - dataset_id: master-telemetry - transformations_config: - - field_key: email - transformation_function: - type: mask - expr: mid - datatype: string - category: pii - mode: Strict - tags: - - tag1 + examples: + example1: + summary: Request Body + value: + id: api.datasets.create + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + type: event + name: sb-telemetry + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: id + dedup_config: + drop_duplicates: true + dedup_key: mid + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + mid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + arrival_format: number + data_type: epoch + eid: + type: string + arrival_format: text + data_type: string + additionalProperties: true + denorm_config: + denorm_fields: + - denorm_key: eid + denorm_out_field: userdata + dataset_id: master-telemetry + transformations_config: + - field_key: email + transformation_function: + type: mask + expr: mid + datatype: string + category: pii + mode: Strict + tags: + - tag1 + example2: + summary: Minimal Request Body + value: + id: api.datasets.create + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + dataset_id: telemetry_events, + type: event + name: sb-telemetry parameters: - name: Content-Type in: header @@ -207,19 +220,33 @@ paths: requestBody: content: application/json: - schema: - type: object - example: - id: api.files.generate-url - ver: v1 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: write + examples: + example1: + summary: Request body for put url + value: + id: api.files.generate-url + ver: v1 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: write + example2: + summary: Request body for get url + value: + id: api.files.generate-url + ver: v1 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: read parameters: - name: Content-Type in: header @@ -2067,100 +2094,116 @@ paths: requestBody: content: application/json: - schema: - type: object - example: - id: api.datasets.update - ver: v1 - ts: '2024-04-10T16:10:50+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - request: - dataset_id: telemetry_record-t4 - version_key: '1721135455988' - name: sb-telemetry - validation_config: - validate: true - mode: Strict - extraction_config: - is_batch_event: true - extraction_key: events + examples: + example1: + summary: Request Body + value: + id: api.datasets.update + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + dataset_id: telemetry_record-t4 + version_key: '1721135455988' + name: sb-telemetry + validation_config: + validate: true + mode: Strict + extraction_config: + is_batch_event: true + extraction_key: events + dedup_config: + drop_duplicates: true + dedup_key: ipid dedup_config: drop_duplicates: true - dedup_key: ipid - dedup_config: - drop_duplicates: true - dedup_key: mid - data_schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - midpid: - type: string - arrival_format: text - data_type: string - miduwi: - type: integer - arrival_format: number - data_type: epoch - mid: - type: string - arrival_format: text - data_type: string - sid: - type: string - arrival_format: text - data_type: string - additionalProperties: true - denorm_config: - denorm_fields: + dedup_key: mid + data_schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + midpid: + type: string + arrival_format: text + data_type: string + miduwi: + type: integer + arrival_format: number + data_type: epoch + mid: + type: string + arrival_format: text + data_type: string + sid: + type: string + arrival_format: text + data_type: string + additionalProperties: true + denorm_config: + denorm_fields: + - value: + denorm_key: eid + denorm_out_field: userdata + action: remove + - value: + denorm_key: eid + denorm_out_field: edata + dataset_id: trip-details + action: upsert + transformations_config: + - value: + field_key: email + transformation_function: + type: mask + expr: mid + datatype: string + category: pii + mode: Strict + action: upsert - value: - denorm_key: eid - denorm_out_field: userdata + field_key: email_id + transformation_function: + type: mask + expr: mid + datatype: string + category: pii + mode: Strict action: remove + tags: [] + connectors_config: - value: - denorm_key: eid - denorm_out_field: edata - dataset_id: trip-details + connector_id: jdbc + connector_config: + source_database_type: postgresql + source_database_host: postgresql-hl.postgresql.svc.cluster.local.master + source_database_port: 5432 + source_database_name: obsrv_sample_datasets_1 + source_database_username: postgres + source_database_pwd: postgres + table: new_york_taxi_data + timestamp-column: tpep_pickup_datetime + batch-size: 100 + max-batches: 2 + operations_config: + polling_interval: periodic + schedule: twice action: upsert - transformations_config: - - value: - field_key: email - transformation_function: - type: mask - expr: mid - datatype: string - category: pii - mode: Strict - action: upsert - - value: - field_key: email_id - transformation_function: - type: mask - expr: mid - datatype: string - category: pii - mode: Strict - action: remove - tags: [] - connectors_config: - - value: - connector_id: jdbc - connector_config: - source_database_type: postgresql - source_database_host: postgresql-hl.postgresql.svc.cluster.local.master - source_database_port: 5432 - source_database_name: obsrv_sample_datasets_1 - source_database_username: postgres - source_database_pwd: postgres - table: new_york_taxi_data - timestamp-column: tpep_pickup_datetime - batch-size: 100 - max-batches: 2 - operations_config: - polling_interval: periodic - schedule: twice - action: upsert + example2: + summary: Minimal request body + value: + id: api.datasets.update + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + dataset_id: telemetry_record-t4 + version_key: '1721135455988' + name: sb-telemetry + validation_config: + validate: true + mode: Strict parameters: - name: Content-Type in: header @@ -2472,18 +2515,51 @@ paths: requestBody: content: application/json: - schema: - type: object - example: - id: api.datasets.list - ver: v1 - ts: '2024-04-10T16:10:50+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - request: - filters: - status: - - Live + examples: + example1: + summary: Filter based on status of type array + value: + id: api.datasets.list + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: + - Live + example2: + summary: No filters provided + value: + id: api.datasets.list + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: {} + example3: + summary: Filter based on status of type string + value: + id: api.datasets.list + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: ReadyToPublish + example4: + summary: Filter based on dataset type + value: + id: api.datasets.list + ver: v1 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: Live + type: master responses: '200': description: OK @@ -4983,4 +5059,829 @@ paths: code: DATASET_LIST_INPUT_INVALID message: >- #properties/params/required must have required property - 'msgid' \ No newline at end of file + 'msgid' + /v2/datasets/dataschema: + post: + tags: + - Dataset api's + summary: Data schema generator + requestBody: + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + data: + - eid: IMPRESSION + ets: 1672657002221 + ver: '3.0' + mid: IMPRESSION:2b5834e196f485c17c4e49d292af43c0 + actor: + id: 0c45959486f579c24854d40a225d6161 + type: User + context: + channel: '01268904781886259221' + pdata: + id: staging.diksha.portal + ver: 5.1.0 + pid: sunbird-portal + env: public + sid: 23850c90-8a8c-11ed-95d0-276800e1048c + did: 0c45959486f579c24854d40a225d6161 + cdata: [] + rollup: + l1: '01268904781886259221' + uid: anonymous + object: {} + tags: + - '01268904781886259221' + edata: + type: view + pageid: login + subtype: pageexit + uri: >- + https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id=portal&state=254efd70-6b89-4f7d-868b-5c957f54174e&redirect_uri=https%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1&scope=openid&response_type=code&version=4 + visits: [] + syncts: 1672657005814 + '@timestamp': '2023-01-02T10:56:45.814Z' + flags: + ex_processed: true + - eid: IMPRESSION + ets: 1672656997928 + ver: '3.0' + mid: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 50263f0f-c2d5-4b15-95f4-5384c537f6cc + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: POST + - url: /v1/org/search + - duration: 0 + - status: OK + - eid: LOG + ets: 1672656998024 + ver: '3.0' + mid: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: 4a340ad0-0665-49b6-a1fa-a581dcac4550 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + type=system, message=EXIT LOG: method : POST, url: + /v1/org/search , For Operation : orgSearch, + params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + errmsg=Invalid value null for parameter hashTagId. + Please provide a valid value., + resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, + err=UOS_ORGSER0017, status=FAILED, + responseCode=400}]}} + params: [] + - eid: LOG + ets: 1672657004961 + ver: '3.0' + mid: f34112c7242a3e3a26f0015796b029c2 + actor: + id: internal + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: Organisation + cdata: + - id: f34112c7242a3e3a26f0015796b029c2 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + ElasticSearchRestHighImpl:search: calling search for + index org_alias, with query = + {"from":0,"size":250,"query":{"bool":{"must":[{"term":{"isTenant.raw":{"value":true,"boost":1.0}}},{"term":{"slug.raw":{"value":"ntp","boost":1.0}}}],"adjust_pure_negative":true,"boost":1.0}},"_source":{"includes":[],"excludes":[]}} + params: [] + - eid: LOG + ets: 1672657006595 + ver: '3.0' + mid: d23ff123-40f0-4262-a69b-b75b46d315a1 + actor: + id: 930a3994-cbe7-4e84-936f-4974096af6f2 + type: Consumer + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: d23ff123-40f0-4262-a69b-b75b46d315a1 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, + type=system, message=ENTRY LOG: method : GET, url: + /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 + , For Operation : getUserRolesById, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006611 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + {eid='LOG', edata={level=trace, + requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, + type=system, message=ENTRY LOG: method : GET, url: + /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , + For Operation : getUserProfileV5, params=[{id=null, + userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}} + params: [] + - eid: LOG + ets: 1672657006620 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user_roles + WHERE userId=?; + params: [] + - eid: LOG + ets: 1672657006645 + ver: '3.0' + mid: 7d944b1c-a906-4082-b42a-905aa6b78a4e + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.learning.service + pid: learner-service + ver: 5.0.0 + env: User + cdata: + - id: 7d944b1c-a906-4082-b42a-905aa6b78a4e + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: '' + params: + - method: GET + - url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 + - duration: 0 + - status: OK + - eid: LOG + ets: 1672657007238 + ver: '3.0' + mid: d4d34fde-c407-efb6-03bd-9f892ca0f114 + actor: + id: 6ab35eea-01fd-4de0-8902-f68722caf859 + type: User + context: + channel: '0126796199493140480' + pdata: + id: staging.sunbird.portal + pid: learner-service + ver: 5.0.0 + env: User + did: d904c90d9f81ddac20141b94ddd606a0 + cdata: + - id: d4d34fde-c407-efb6-03bd-9f892ca0f114 + type: Request + rollup: {} + edata: + level: info + type: Api_access + message: >- + Cassandra query : SELECT * FROM sunbird.user WHERE + id=?; + params: [] + config: + dataset: financial_transactions + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + example: + id: api.datasets.dataschema + ver: v1 + ts: '2024-07-22T12:32:50+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1309aea0-9a97-46e9-bc5e-a16a8a7fb624 + responseCode: OK + result: + schema: + $schema: https://json-schema.org/draft/2020-12/schema + type: object + properties: + eid: + type: string + arrival_format: text + data_type: string + ets: + type: integer + suggestions: + - message: >- + The Property 'ets' appears to be 'epoch' format + type. + severity: '' + path: properties.ets + arrival_format: number + data_type: epoch + ver: + type: string + arrival_format: text + data_type: string + mid: + type: string + suggestions: + - message: >- + The Property 'mid' appears to be 'uuid' format + type. + advice: Suggest to not to index the high cardinal columns + resolutionType: DEDUP + severity: LOW + path: properties.mid + arrival_format: text + data_type: string + actor: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'actor.id' appears to be 'uuid' + format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.actor.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + context: + type: object + properties: + channel: + type: string + arrival_format: text + data_type: string + pdata: + type: object + properties: + id: + type: string + arrival_format: text + data_type: string + ver: + type: string + arrival_format: text + data_type: string + pid: + type: string + arrival_format: text + data_type: string + arrival_format: object + data_type: object + env: + type: string + arrival_format: text + data_type: string + sid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'sid'. The property sid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.sid + - message: >- + The Property 'context.sid' appears to be + 'uuid' format type. + advice: >- + Suggest to not to index the high cardinal + columns + resolutionType: DEDUP + severity: LOW + path: properties.context.properties.sid + arrival_format: text + data_type: string + did: + type: string + arrival_format: text + data_type: string + cdata: + type: array + items: + type: object + properties: + id: + type: string + suggestions: + - message: >- + The Property 'context.cdata[*].id' + appears to be 'uuid' format type. + advice: >- + Suggest to not to index the high + cardinal columns + resolutionType: DEDUP + severity: LOW + path: >- + properties.context.properties.cdata.items.properties.id + arrival_format: text + data_type: string + type: + type: string + arrival_format: text + data_type: string + arrival_format: array + data_type: array + rollup: + type: object + properties: + l1: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at + property: 'l1'. The property l1: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System + has updated the property schema to + optional + resolutionType: OPTIONAL + severity: MEDIUM + path: >- + properties.context.properties.rollup.properties.l1 + arrival_format: text + data_type: string + arrival_format: object + data_type: object + uid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uid'. The property uid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.context.properties.uid + arrival_format: text + data_type: string + arrival_format: object + data_type: object + object: + type: object + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'object'. The property object: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.object + arrival_format: object + data_type: object + tags: + type: array + items: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'tags'. The property tags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.tags + arrival_format: array + data_type: array + edata: + type: object + properties: + type: + type: string + arrival_format: text + data_type: string + pageid: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'pageid'. The property pageid: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.pageid + arrival_format: text + data_type: string + subtype: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'subtype'. The property subtype: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.subtype + arrival_format: text + data_type: string + uri: + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'uri'. The property uri: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.uri + - message: >- + The Property 'edata.uri' appears to be 'uri' + format type. + severity: '' + path: properties.edata.properties.uri + arrival_format: text + data_type: string + visits: + type: array + items: {} + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'visits'. The property visits: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.edata.properties.visits + arrival_format: array + data_type: array + level: + type: string + arrival_format: text + data_type: string + message: + type: string + arrival_format: text + data_type: string + params: + type: array + items: {} + arrival_format: array + data_type: array + arrival_format: object + data_type: object + syncts: + type: integer + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'syncts'. The property syncts: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.syncts + - message: >- + The Property 'syncts' appears to be 'epoch' format + type. + severity: '' + path: properties.syncts + arrival_format: number + data_type: epoch + '@timestamp': + type: string + suggestions: + - message: >- + Conflict in the Schema Generation at property: + '@timestamp'. The property @timestamp: only 1 + time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.@timestamp + - message: >- + The Property '@timestamp' appears to be + 'date-time' format type. + advice: The System can index all data on this column + resolutionType: INDEX + severity: LOW + path: properties.@timestamp + arrival_format: text + data_type: date-time + flags: + type: object + properties: + ex_processed: + type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'ex_processed'. The property ex_processed: + only 1 time(s) appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags.properties.ex_processed + arrival_format: boolean + data_type: boolean + suggestions: + - message: >- + Conflict in the Schema Generation at property: + 'flags'. The property flags: only 1 time(s) + appeared + advice: >- + The Property looks to be Optional. System has + updated the property schema to optional + resolutionType: OPTIONAL + severity: MEDIUM + path: properties.flags + arrival_format: object + data_type: object + additionalProperties: true + configurations: + indexConfiguration: + index: + Event Arrival Time: obsrv_meta.syncts + rollupSuggestions: + summary: + mid: + path: $.mid + cardinality: 67 + index: false + actor.id: + path: $.actor.properties.id + cardinality: 56 + index: false + context.sid: + path: $.context.properties.sid + cardinality: 11 + index: true + edata.uri: + path: $.edata.properties.uri + cardinality: 11 + index: true + context.cdata[*].id: + path: $.context.properties.cdata.items.properties.id + cardinality: 62 + index: false + processing: + dedupKeys: + - mid + - context.cdata[*].id + - actor.id + dropDuplicates: + - 'Yes' + - 'No' + dataMappings: + text: + arrival_format: + - string + store_format: + string: + jsonSchema: string + datasource: string + date-time: + jsonSchema: string + datasource: string + date: + jsonSchema: string + datasource: string + boolean: + jsonSchema: string + datasource: boolean + epoch: + jsonSchema: string + datasource: integer + long: + jsonSchema: string + datasource: long + double: + jsonSchema: string + datasource: double + bigdecimal: + jsonSchema: string + datasource: double + integer: + jsonSchema: string + datasource: long + number: + arrival_format: + - number + - integer + store_format: + integer: + jsonSchema: integer + datasource: long + float: + jsonSchema: number + datasource: double + long: + jsonSchema: integer + datasource: long + double: + jsonSchema: number + datasource: double + bigdecimal: + jsonSchema: number + datasource: double + epoch: + jsonSchema: integer + datasource: long + number: + jsonSchema: number + datasource: double + object: + arrival_format: + - object + store_format: + object: + jsonSchema: object + datasource: json + array: + arrival_format: + - array + store_format: + array: + jsonSchema: array + datasource: array + boolean: + arrival_format: + - boolean + store_format: + boolean: + jsonSchema: boolean + datasource: boolean + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: Invalid request body' + value: + id: api.datasets.dataschema + ver: v1 + ts: '2024-07-22T12:33:47+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: bbcc86c2-042d-4f77-bb6e-e1c9116df570 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'data' + example-1: + summary: 'Failure: Invalid request (config not provided)' + value: + id: api.datasets.dataschema + ver: v1 + ts: '2024-07-22T12:35:36+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 1f856c5e-37f0-41e9-96fb-642471228da2 + responseCode: BAD_REQUEST + result: {} + error: + code: DATA_SCHEMA_INVALID_INPUT + message: >- + #properties/request/required must have required property + 'config' \ No newline at end of file From 2c6baa15907e2ceb4ee2e5c2db30c45a4365f291 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 22 Jul 2024 13:22:05 +0530 Subject: [PATCH 045/235] #OBS-I126 : updated order --- .../swagger-doc/v2_updated_doc_openapi.yml | 2009 ++--------------- 1 file changed, 133 insertions(+), 1876 deletions(-) diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml index fef21941..f51caf14 100644 --- a/api-service/swagger-doc/v2_updated_doc_openapi.yml +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -1,6 +1,6 @@ openapi: 3.0.0 info: - title: V2 docs + title: Obsrv api documentation version: 1.0.0 servers: - url: localhost:3000 @@ -212,1880 +212,6 @@ paths: error: code: DATASET_EXISTS message: Dataset Already exists with id:telemetry_record-t4 - /v2/files/generate-url: - post: - tags: - - Dataset api's - summary: File generate url - requestBody: - content: - application/json: - examples: - example1: - summary: Request body for put url - value: - id: api.files.generate-url - ver: v1 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: write - example2: - summary: Request body for get url - value: - id: api.files.generate-url - ver: v1 - ts: '2024-04-19T12:58:47+05:30' - params: - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - request: - files: - - telemetry.json - - school_data.json - access: read - parameters: - - name: Content-Type - in: header - schema: - type: string - example: application/json - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Success: Generate put url' - value: - id: api.files.generate-url - ver: v1 - ts: '2024-07-16T08:26:19+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 5306f309-4a15-458e-89e2-29d8ac0835d4 - responseCode: OK - result: - - filePath: >- - test-connector/api-service/user_uploads/telemetry_10d595.json - fileName: telemetry.json - preSignedUrl: >- - https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject - - filePath: >- - test-connector/api-service/user_uploads/school_data_33109a.json - fileName: school_data.json - preSignedUrl: >- - https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject - example-1: - summary: 'Success: Generate get url' - value: - id: api.files.generate-url - ver: v1 - ts: '2024-07-16T09:31:40+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: 009c0b2d-8acd-40b0-a807-bbacf9242771 - responseCode: OK - result: - - filePath: test-connector/api-service/user_uploads/telemetry.json - fileName: telemetry.json - preSignedUrl: >- - https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject - - filePath: >- - test-connector/api-service/user_uploads/school_data.json - fileName: school_data.json - preSignedUrl: >- - https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject - '400': - description: Bad Request - content: - application/json: - schema: - type: object - examples: - example-0: - summary: 'Failure: limit exceeds' - value: - id: api.files.generate-url - ver: v1 - ts: '2024-07-16T08:33:04+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: d3a606ca-47d0-4746-95a1-c8692e749959 - responseCode: BAD_REQUEST - error: - code: FILES_URL_GENERATION_LIMIT_EXCEED - message: 'Pre-signed URL generation failed: limit exceeded.' - trace: '' - example-1: - summary: 'Failure: Invalid request' - value: - id: api.files.generate-url - ver: v1 - ts: '2024-07-16T09:31:10+05:30' - params: - status: FAILED - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 - resmsgid: c3e9da1c-09f3-4a3b-84ec-a19efc68b856 - responseCode: BAD_REQUEST - error: - code: FILES_GENERATE_URL_INPUT_INVALID - message: >- - #properties/request/properties/access/enum must be equal - to one of the allowed values - trace: '' - /dataset/v1/dataschema: - post: - tags: - - Dataset api's - summary: Data schema generator - requestBody: - content: - application/json: - schema: - type: object - example: - data: - - tripID: b97b0dbd-1463-4a42-99ff-211fc389464c - VendorID: '1' - tpep_pickup_datetime: '2024-02-12 00:22:30' - tpep_dropoff_datetime: '2023-09-14 00:40:50' - passenger_count: '2' - trip_distance: '2.80' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '249' - DOLocationID: '162' - payment_type: '2' - primary_passenger: - email: Raquel.Kunde@gmail.com - mobile: 310-255-4865 x1413 - fare_details: - fare_amount: '13.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '14.8' - congestion_surcharge: '' - - tripID: a0520d95-1ae2-4d94-a6d2-0e35857e2b3c - VendorID: '1' - tpep_pickup_datetime: '2023-07-01 00:47:00' - tpep_dropoff_datetime: '2024-01-25 00:52:18' - passenger_count: '1' - trip_distance: '1.00' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '236' - DOLocationID: '140' - payment_type: '2' - primary_passenger: - email: Willy15@gmail.com - mobile: 474-817-2801 x633 - fare_details: - fare_amount: '6' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '7.3' - congestion_surcharge: '' - - tripID: 6248522a-ffbb-4d73-8b89-75a0e5310f25 - VendorID: '1' - tpep_pickup_datetime: '2023-03-19 00:55:39' - tpep_dropoff_datetime: '2023-03-16 01:03:06' - passenger_count: '4' - trip_distance: '1.90' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '140' - DOLocationID: '162' - payment_type: '2' - primary_passenger: - email: Agustina74@yahoo.com - mobile: 1-533-609-5857 x24749 - fare_details: - fare_amount: '8' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.3' - congestion_surcharge: '' - - tripID: 3502a658-1bf8-4d62-a180-f1560d088d36 - VendorID: '2' - tpep_pickup_datetime: '2023-06-02 00:28:37' - tpep_dropoff_datetime: '2024-01-26 00:31:37' - passenger_count: '1' - trip_distance: '.76' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '142' - DOLocationID: '239' - payment_type: '1' - primary_passenger: - email: Shemar97@hotmail.com - mobile: (738) 409-8443 x5839 - fare_details: - fare_amount: '4.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '2' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '7.8' - congestion_surcharge: '' - - tripID: 245a42b8-be3f-49a7-b82b-755a60c0fe90 - VendorID: '2' - tpep_pickup_datetime: '2023-03-16 00:33:19' - tpep_dropoff_datetime: '2023-06-17 00:46:44' - passenger_count: '1' - trip_distance: '3.60' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '239' - DOLocationID: '68' - payment_type: '1' - primary_passenger: - email: Alvis.Kshlerin77@hotmail.com - mobile: 1-487-796-9469 x057 - fare_details: - fare_amount: '13.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '2' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '16.8' - congestion_surcharge: '' - - tripID: 4e351ad9-b554-49fe-bdeb-351ced4a5fbc - VendorID: '2' - tpep_pickup_datetime: '2023-05-11 00:59:05' - tpep_dropoff_datetime: '2023-06-21 01:19:25' - passenger_count: '2' - trip_distance: '3.80' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '246' - DOLocationID: '43' - payment_type: '2' - primary_passenger: - email: Nicole.White@hotmail.com - mobile: 687-578-1535 x50831 - fare_details: - fare_amount: '16.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '17.8' - congestion_surcharge: '' - - tripID: 9f24c946-4f7d-4e2b-b0f0-34c11b61b97e - VendorID: '2' - tpep_pickup_datetime: '2023-03-04 00:11:27' - tpep_dropoff_datetime: '2024-01-14 00:46:29' - passenger_count: '5' - trip_distance: '21.42' - RatecodeID: '2' - store_and_fwd_flag: 'N' - PULocationID: '132' - DOLocationID: '87' - payment_type: '1' - primary_passenger: - email: Talia.Denesik@hotmail.com - mobile: 727-537-1685 x107 - fare_details: - fare_amount: '52' - extra: '0' - mta_tax: '0.5' - tip_amount: '11.71' - tolls_amount: '5.76' - improvement_surcharge: '0.3' - total_amount: '70.27' - congestion_surcharge: '' - - tripID: 4622df23-a8f1-482f-9310-76d6fb772b9a - VendorID: '2' - tpep_pickup_datetime: '2023-04-02 00:19:27' - tpep_dropoff_datetime: '2023-03-06 01:03:21' - passenger_count: '1' - trip_distance: '10.84' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '114' - DOLocationID: '198' - payment_type: '2' - primary_passenger: - email: Alice43@gmail.com - mobile: 579-402-1634 x87762 - fare_details: - fare_amount: '37.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '38.8' - congestion_surcharge: '' - - tripID: 8f8211bf-9693-4b36-817f-d9c1c7253691 - VendorID: '1' - tpep_pickup_datetime: '2023-05-12 00:11:49' - tpep_dropoff_datetime: '2023-02-28 00:23:07' - passenger_count: '2' - trip_distance: '3.60' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '161' - DOLocationID: '41' - payment_type: '1' - primary_passenger: - email: Mike88@gmail.com - mobile: (501) 617-2020 x4697 - fare_details: - fare_amount: '12.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '2.5' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '16.3' - congestion_surcharge: '' - - tripID: 6a102595-7db4-4184-bd80-ce249b784f0b - VendorID: '1' - tpep_pickup_datetime: '2023-12-02 00:38:41' - tpep_dropoff_datetime: '2024-02-01 01:08:52' - passenger_count: '4' - trip_distance: '3.20' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '161' - DOLocationID: '249' - payment_type: '2' - primary_passenger: - email: Vella.Armstrong71@gmail.com - mobile: 978.794.7934 x32048 - fare_details: - fare_amount: '18.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '19.8' - congestion_surcharge: '' - - tripID: ceced196-7da9-4bac-83ad-fe0129098574 - VendorID: '2' - tpep_pickup_datetime: '2024-02-20 00:14:19' - tpep_dropoff_datetime: '2023-11-22 00:17:03' - passenger_count: '2' - trip_distance: '.78' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '236' - DOLocationID: '236' - payment_type: '1' - primary_passenger: - email: Keven_Kihn@hotmail.com - mobile: 1-323-467-0737 x980 - fare_details: - fare_amount: '4.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.16' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '6.96' - congestion_surcharge: '' - - tripID: 23a467f1-de9b-474f-a60a-3d103ebeba04 - VendorID: '2' - tpep_pickup_datetime: '2023-04-03 00:20:57' - tpep_dropoff_datetime: '2023-11-12 00:30:01' - passenger_count: '1' - trip_distance: '1.71' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '236' - DOLocationID: '142' - payment_type: '2' - primary_passenger: - email: Kiera.Kling@hotmail.com - mobile: 704-373-7606 x93095 - fare_details: - fare_amount: '8.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.8' - congestion_surcharge: '' - - tripID: 15c7f4d7-fa74-41ee-98d5-7c00f30bb366 - VendorID: '2' - tpep_pickup_datetime: '2023-07-25 00:36:13' - tpep_dropoff_datetime: '2023-08-19 00:46:08' - passenger_count: '2' - trip_distance: '2.28' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '43' - DOLocationID: '151' - payment_type: '1' - primary_passenger: - email: Laura_Lind99@hotmail.com - mobile: (846) 893-6673 x69711 - fare_details: - fare_amount: '9.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '2.16' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '12.96' - congestion_surcharge: '' - - tripID: 796dfa5d-e16d-49c9-b41c-f4223a8fedd9 - VendorID: '2' - tpep_pickup_datetime: '2023-08-14 00:56:36' - tpep_dropoff_datetime: '2023-07-08 01:04:49' - passenger_count: '2' - trip_distance: '1.57' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '238' - DOLocationID: '41' - payment_type: '1' - primary_passenger: - email: Melvin68@yahoo.com - mobile: 1-436-319-7744 x1743 - fare_details: - fare_amount: '8' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.86' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '11.16' - congestion_surcharge: '' - - tripID: b8b56888-8bb4-4981-b669-2a56d563b25f - VendorID: '2' - tpep_pickup_datetime: '2023-06-23 00:27:41' - tpep_dropoff_datetime: '2024-02-12 00:42:43' - passenger_count: '1' - trip_distance: '2.80' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '113' - DOLocationID: '229' - payment_type: '2' - primary_passenger: - email: Mauricio8@gmail.com - mobile: 1-298-756-7810 x0828 - fare_details: - fare_amount: '12' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '13.3' - congestion_surcharge: '' - - tripID: 4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2 - VendorID: '2' - tpep_pickup_datetime: '2023-09-30 00:46:37' - tpep_dropoff_datetime: '2023-04-07 00:53:37' - passenger_count: '1' - trip_distance: '1.18' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '141' - DOLocationID: '262' - payment_type: '1' - primary_passenger: - email: Bailee.Roob80@gmail.com - mobile: 753.766.7597 x58905 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.66' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.96' - congestion_surcharge: '' - - tripID: dc5fb067-ee76-4c74-8d53-913518d7de3c - VendorID: '1' - tpep_pickup_datetime: '2023-09-07 00:21:23' - tpep_dropoff_datetime: '2023-08-14 00:26:07' - passenger_count: '2' - trip_distance: '.40' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '233' - DOLocationID: '233' - payment_type: '2' - primary_passenger: - email: Francisco70@yahoo.com - mobile: 1-591-565-3358 x04096 - fare_details: - fare_amount: '4.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '5.8' - congestion_surcharge: '' - - tripID: d1008fa4-1017-43e5-a7cd-7fcc235d82e5 - VendorID: '1' - tpep_pickup_datetime: '2023-08-13 00:49:45' - tpep_dropoff_datetime: '2023-08-30 00:56:01' - passenger_count: '1' - trip_distance: '1.00' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '113' - DOLocationID: '79' - payment_type: '2' - primary_passenger: - email: Charity_Jacobs58@gmail.com - mobile: 1-713-793-4442 x6226 - fare_details: - fare_amount: '6' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '7.3' - congestion_surcharge: '' - - tripID: a4d907fa-cad9-41fa-a25f-cd3fac14d0d9 - VendorID: '1' - tpep_pickup_datetime: '2023-10-27 00:56:42' - tpep_dropoff_datetime: '2023-03-23 01:26:09' - passenger_count: '4' - trip_distance: '6.00' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '79' - DOLocationID: '75' - payment_type: '1' - primary_passenger: - email: Bridie61@yahoo.com - mobile: (632) 512-8857 - fare_details: - fare_amount: '24' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '5.05' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '30.35' - congestion_surcharge: '' - - tripID: 28c84ea3-4129-4114-b0ee-7bb39cb9613f - VendorID: '2' - tpep_pickup_datetime: '2024-03-07 00:10:50' - tpep_dropoff_datetime: '2023-06-10 00:17:46' - passenger_count: '5' - trip_distance: '1.32' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '42' - DOLocationID: '41' - payment_type: '2' - primary_passenger: - email: Vickie.Franey93@gmail.com - mobile: 874.663.4243 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '8.3' - congestion_surcharge: '' - - tripID: 1218ae38-7474-4a4b-9343-e3d0fe6da9c6 - VendorID: '2' - tpep_pickup_datetime: '2023-07-18 00:32:36' - tpep_dropoff_datetime: '2023-06-16 00:54:08' - passenger_count: '3' - trip_distance: '3.65' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '43' - DOLocationID: '224' - payment_type: '2' - primary_passenger: - email: Alexandro.Prohaska40@hotmail.com - mobile: (413) 378-0852 x100 - fare_details: - fare_amount: '15.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '16.8' - congestion_surcharge: '' - - tripID: 0a5fc580-5a28-4153-b37f-a2ecc4ebc777 - VendorID: '1' - tpep_pickup_datetime: '2023-05-06 00:32:24' - tpep_dropoff_datetime: '2023-11-29 00:33:21' - passenger_count: '0' - trip_distance: '5.30' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '50' - DOLocationID: '50' - payment_type: '1' - primary_passenger: - email: Veda_Stamm@hotmail.com - mobile: 596-555-2936 x826 - fare_details: - fare_amount: '2.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0.75' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '4.55' - congestion_surcharge: '' - - tripID: 89856e17-912a-40e6-8379-f1d75a066878 - VendorID: '1' - tpep_pickup_datetime: '2023-05-30 00:36:37' - tpep_dropoff_datetime: '2024-02-14 00:42:40' - passenger_count: '1' - trip_distance: '1.40' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '50' - DOLocationID: '239' - payment_type: '1' - primary_passenger: - email: Jed60@hotmail.com - mobile: (761) 368-4573 - fare_details: - fare_amount: '6.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.55' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.35' - congestion_surcharge: '' - - tripID: fece3c74-e914-46c6-8b84-88470d9c7b3a - VendorID: '1' - tpep_pickup_datetime: '2023-09-14 00:44:29' - tpep_dropoff_datetime: '2023-03-22 00:48:05' - passenger_count: '3' - trip_distance: '.60' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '238' - DOLocationID: '238' - payment_type: '2' - primary_passenger: - email: Juwan_Schimmel@yahoo.com - mobile: 1-725-659-7644 - fare_details: - fare_amount: '4.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '5.8' - congestion_surcharge: '' - - tripID: 9142ffc4-db10-4faf-ad6a-b15595e02408 - VendorID: '1' - tpep_pickup_datetime: '2024-02-14 00:52:53' - tpep_dropoff_datetime: '2023-09-29 01:27:03' - passenger_count: '2' - trip_distance: '14.40' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '238' - DOLocationID: '98' - payment_type: '2' - primary_passenger: - email: Pinkie13@hotmail.com - mobile: (991) 905-4690 - fare_details: - fare_amount: '42' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '5.76' - improvement_surcharge: '0.3' - total_amount: '49.06' - congestion_surcharge: '' - - tripID: ac62b151-e94c-473f-b461-c25f7035e8ac - VendorID: '2' - tpep_pickup_datetime: '2023-08-01 00:28:12' - tpep_dropoff_datetime: '2023-03-08 00:34:55' - passenger_count: '1' - trip_distance: '1.43' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '14' - DOLocationID: '228' - payment_type: '2' - primary_passenger: - email: Esteban_Ondricka@yahoo.com - mobile: (697) 411-7922 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '8.3' - congestion_surcharge: '' - - tripID: 707e720d-afdd-4af9-9469-02fd5052ed9f - VendorID: '2' - tpep_pickup_datetime: '2023-10-10 00:55:16' - tpep_dropoff_datetime: '2023-03-25 01:01:05' - passenger_count: '1' - trip_distance: '1.00' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '11' - DOLocationID: '22' - payment_type: '2' - primary_passenger: - email: Kristin.Baumbach-Ferry@hotmail.com - mobile: 964-998-1199 - fare_details: - fare_amount: '6' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '7.3' - congestion_surcharge: '' - - tripID: e75c5819-6f52-4c57-a6ea-85b79935821b - VendorID: '1' - tpep_pickup_datetime: '2023-07-11 00:29:10' - tpep_dropoff_datetime: '2023-08-10 01:23:09' - passenger_count: '2' - trip_distance: '11.00' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '181' - DOLocationID: '239' - payment_type: '1' - primary_passenger: - email: Dorothea.Kulas@gmail.com - mobile: 354.367.7954 - fare_details: - fare_amount: '40' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '10.33' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '51.63' - congestion_surcharge: '' - - tripID: d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d - VendorID: '2' - tpep_pickup_datetime: '2024-01-12 00:20:30' - tpep_dropoff_datetime: '2023-11-09 00:36:07' - passenger_count: '1' - trip_distance: '1.50' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '161' - DOLocationID: '229' - payment_type: '2' - primary_passenger: - email: Jazmyn_Corwin@gmail.com - mobile: 642.790.7928 x83713 - fare_details: - fare_amount: '10.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '11.8' - congestion_surcharge: '' - - tripID: 6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba - VendorID: '2' - tpep_pickup_datetime: '2024-02-04 00:42:50' - tpep_dropoff_datetime: '2024-03-03 00:51:48' - passenger_count: '1' - trip_distance: '.72' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '162' - DOLocationID: '43' - payment_type: '2' - primary_passenger: - email: Mazie18@hotmail.com - mobile: 1-641-639-4170 x249 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '8.3' - congestion_surcharge: '' - - tripID: 18578bdf-e169-4d3d-ae08-b6320bb04e29 - VendorID: '2' - tpep_pickup_datetime: '2023-04-19 00:53:39' - tpep_dropoff_datetime: '2023-05-12 00:51:48' - passenger_count: '1' - trip_distance: '2.60' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '43' - DOLocationID: '137' - payment_type: '1' - primary_passenger: - email: Maud.Collins@gmail.com - mobile: 214.847.9872 - fare_details: - fare_amount: '13' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '2.14' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '16.44' - congestion_surcharge: '' - - tripID: b44c85e8-ebd2-481f-a9d5-52124dec673a - VendorID: '1' - tpep_pickup_datetime: '2023-02-25 00:16:33' - tpep_dropoff_datetime: '2023-09-01 00:23:16' - passenger_count: '1' - trip_distance: '1.50' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '236' - DOLocationID: '237' - payment_type: '1' - primary_passenger: - email: Domenick_Pagac@yahoo.com - mobile: 742.839.7610 x189 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.65' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.95' - congestion_surcharge: '' - - tripID: dc355724-1226-4dbf-ace7-f5c34e39b7e7 - VendorID: '1' - tpep_pickup_datetime: '2023-07-23 00:24:43' - tpep_dropoff_datetime: '2023-06-26 00:30:13' - passenger_count: '1' - trip_distance: '1.00' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '237' - DOLocationID: '140' - payment_type: '2' - primary_passenger: - email: Kathryne1@yahoo.com - mobile: (900) 624-9537 - fare_details: - fare_amount: '6' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '7.3' - congestion_surcharge: '' - - tripID: e65bc848-a894-4fb7-b889-dd7d812ab793 - VendorID: '1' - tpep_pickup_datetime: '2023-12-01 00:34:18' - tpep_dropoff_datetime: '2023-10-11 00:38:47' - passenger_count: '1' - trip_distance: '.60' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '140' - DOLocationID: '237' - payment_type: '1' - primary_passenger: - email: Hilton.Jacobi@gmail.com - mobile: 1-637-451-2136 - fare_details: - fare_amount: '5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.25' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '7.55' - congestion_surcharge: '' - - tripID: 32bf8dee-cd36-4c9e-befb-be32256653e0 - VendorID: '1' - tpep_pickup_datetime: '2024-02-01 00:39:55' - tpep_dropoff_datetime: '2023-03-31 00:51:01' - passenger_count: '1' - trip_distance: '1.90' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '237' - DOLocationID: '238' - payment_type: '1' - primary_passenger: - email: Ole.Lindgren@hotmail.com - mobile: (728) 501-8337 x72810 - fare_details: - fare_amount: '9.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '3.2' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '14' - congestion_surcharge: '' - - tripID: 1e7bc9a0-ccc6-4855-8650-a46e8695c1a2 - VendorID: '1' - tpep_pickup_datetime: '2023-08-24 00:52:47' - tpep_dropoff_datetime: '2023-12-12 00:59:52' - passenger_count: '1' - trip_distance: '1.90' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '238' - DOLocationID: '237' - payment_type: '1' - primary_passenger: - email: Jazmin.Pollich-Little81@gmail.com - mobile: 517-815-1765 - fare_details: - fare_amount: '8' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.85' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '11.15' - congestion_surcharge: '' - - tripID: b81e5976-6581-4cad-914d-b1bf3a007c7b - VendorID: '1' - tpep_pickup_datetime: '2023-08-24 00:24:43' - tpep_dropoff_datetime: '2023-05-19 00:37:30' - passenger_count: '1' - trip_distance: '1.30' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '162' - DOLocationID: '107' - payment_type: '1' - primary_passenger: - email: Jake_Heidenreich52@gmail.com - mobile: 300-566-1457 x9243 - fare_details: - fare_amount: '9.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '2.15' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '12.95' - congestion_surcharge: '' - - tripID: beae8765-a627-4af7-ac5b-a5ba96f6d54f - VendorID: '1' - tpep_pickup_datetime: '2023-12-29 00:47:22' - tpep_dropoff_datetime: '2023-10-17 00:55:16' - passenger_count: '1' - trip_distance: '.80' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '107' - DOLocationID: '79' - payment_type: '1' - primary_passenger: - email: Mitchell.Green64@hotmail.com - mobile: 1-279-558-0312 x75465 - fare_details: - fare_amount: '6.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.55' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.35' - congestion_surcharge: '' - - tripID: a59b934c-583c-4fa3-a5cb-15041379e8da - VendorID: '1' - tpep_pickup_datetime: '2023-09-17 00:59:36' - tpep_dropoff_datetime: '2023-09-12 01:28:32' - passenger_count: '1' - trip_distance: '3.80' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '113' - DOLocationID: '142' - payment_type: '1' - primary_passenger: - email: Esther.Hintz@hotmail.com - mobile: 546-446-7171 x1903 - fare_details: - fare_amount: '19.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '4.15' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '24.95' - congestion_surcharge: '' - - tripID: 7ed9ad81-42cb-436e-93c5-d88d7245493f - VendorID: '1' - tpep_pickup_datetime: '2023-12-29 00:18:04' - tpep_dropoff_datetime: '2023-10-04 00:25:32' - passenger_count: '1' - trip_distance: '1.30' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '161' - DOLocationID: '137' - payment_type: '1' - primary_passenger: - email: Carolanne_Buckridge@gmail.com - mobile: (972) 507-5995 x716 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '8.3' - congestion_surcharge: '' - - tripID: 4c325355-2689-482f-8107-ece7515ddc33 - VendorID: '1' - tpep_pickup_datetime: '2023-08-17 00:40:33' - tpep_dropoff_datetime: '2023-03-17 00:44:25' - passenger_count: '2' - trip_distance: '.40' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '164' - DOLocationID: '170' - payment_type: '4' - primary_passenger: - email: Willie.Zieme44@hotmail.com - mobile: 813.930.8291 x27037 - fare_details: - fare_amount: '4.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '5.8' - congestion_surcharge: '' - - tripID: 5e153af5-1e69-4acc-bd70-25b7d6e9856a - VendorID: '1' - tpep_pickup_datetime: '2023-09-23 00:59:13' - tpep_dropoff_datetime: '2023-10-21 01:01:44' - passenger_count: '6' - trip_distance: '.50' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '161' - DOLocationID: '161' - payment_type: '3' - primary_passenger: - email: Abe.Kassulke5@yahoo.com - mobile: 1-994-727-5845 x8326 - fare_details: - fare_amount: '4' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '5.3' - congestion_surcharge: '' - - tripID: eef4af05-c391-4a67-9ed9-0a6a2c23645c - VendorID: '1' - tpep_pickup_datetime: '2024-03-11 00:10:40' - tpep_dropoff_datetime: '2023-03-16 00:27:11' - passenger_count: '2' - trip_distance: '4.30' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '229' - DOLocationID: '223' - payment_type: '2' - primary_passenger: - email: Clifford.Fisher@yahoo.com - mobile: 1-934-478-8531 x33207 - fare_details: - fare_amount: '16' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '17.3' - congestion_surcharge: '' - - tripID: a790d399-dbbc-4483-805b-0fb001b6c6b7 - VendorID: '2' - tpep_pickup_datetime: '2023-06-27 00:25:03' - tpep_dropoff_datetime: '2024-03-07 01:02:07' - passenger_count: '6' - trip_distance: '3.58' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '211' - DOLocationID: '48' - payment_type: '2' - primary_passenger: - email: Carson.Hartmann-Lynch59@gmail.com - mobile: (856) 507-5220 x738 - fare_details: - fare_amount: '23' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '24.3' - congestion_surcharge: '' - - tripID: 10fb7db8-edee-4686-95e7-d07a17761fe9 - VendorID: '1' - tpep_pickup_datetime: '2024-02-17 00:26:54' - tpep_dropoff_datetime: '2023-08-06 00:49:47' - passenger_count: '2' - trip_distance: '2.10' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '158' - DOLocationID: '107' - payment_type: '2' - primary_passenger: - email: Brandyn.Powlowski43@yahoo.com - mobile: (972) 869-2846 - fare_details: - fare_amount: '14.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '15.8' - congestion_surcharge: '' - - tripID: b575d34f-8874-4b67-8918-293cccec8558 - VendorID: '2' - tpep_pickup_datetime: '2023-03-04 00:18:00' - tpep_dropoff_datetime: '2023-11-02 00:26:10' - passenger_count: '1' - trip_distance: '1.11' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '13' - DOLocationID: '231' - payment_type: '1' - primary_passenger: - email: Lelia71@hotmail.com - mobile: 298-955-0204 x145 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '8.3' - congestion_surcharge: '' - - tripID: 4524ce02-c41d-4e83-82a1-e41b00d97dab - VendorID: '2' - tpep_pickup_datetime: '2024-01-25 00:33:25' - tpep_dropoff_datetime: '2023-10-29 01:07:18' - passenger_count: '1' - trip_distance: '5.63' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '113' - DOLocationID: '238' - payment_type: '1' - primary_passenger: - email: Magali_Mohr90@gmail.com - mobile: 1-339-745-7996 x126 - fare_details: - fare_amount: '24' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '5.06' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '30.36' - congestion_surcharge: '' - - tripID: 0a4d6094-f334-4e18-a1e9-1001820b6f89 - VendorID: '1' - tpep_pickup_datetime: '2024-01-26 00:11:00' - tpep_dropoff_datetime: '2023-08-01 00:15:28' - passenger_count: '1' - trip_distance: '.80' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '79' - DOLocationID: '79' - payment_type: '1' - primary_passenger: - email: Sydney.Sanford71@yahoo.com - mobile: 557.212.3262 x589 - fare_details: - fare_amount: '5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.26' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '7.56' - congestion_surcharge: '' - - tripID: 9245af5e-632e-4f21-9060-88522728ab73 - VendorID: '1' - tpep_pickup_datetime: '2023-09-07 00:17:57' - tpep_dropoff_datetime: '2023-09-14 00:27:43' - passenger_count: '1' - trip_distance: '3.70' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '107' - DOLocationID: '87' - payment_type: '1' - primary_passenger: - email: Deanna15@hotmail.com - mobile: 927-327-6309 x16689 - fare_details: - fare_amount: '13' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '2.85' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '17.15' - congestion_surcharge: '' - - tripID: 4dcce454-046c-4b67-bd80-ff773a3c3d96 - VendorID: '1' - tpep_pickup_datetime: '2023-08-16 00:35:11' - tpep_dropoff_datetime: '2023-05-07 00:58:40' - passenger_count: '1' - trip_distance: '5.40' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '261' - DOLocationID: '142' - payment_type: '1' - primary_passenger: - email: Jeramie33@yahoo.com - mobile: 757-419-8948 x7985 - fare_details: - fare_amount: '20.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '5.45' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '27.25' - congestion_surcharge: '' - - tripID: d5808aed-5e8e-447f-8c42-521f8472a24d - VendorID: '1' - tpep_pickup_datetime: '2024-01-23 00:12:48' - tpep_dropoff_datetime: '2023-07-16 00:23:48' - passenger_count: '1' - trip_distance: '1.80' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '211' - DOLocationID: '232' - payment_type: '2' - primary_passenger: - email: Jarrod_Bergstrom@hotmail.com - mobile: 388-916-2388 x911 - fare_details: - fare_amount: '9.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '10.8' - congestion_surcharge: '' - - tripID: faf99a1d-127f-432a-bdb9-39c91a205ec3 - VendorID: '1' - tpep_pickup_datetime: '2023-08-12 00:31:53' - tpep_dropoff_datetime: '2023-09-30 00:47:26' - passenger_count: '1' - trip_distance: '2.10' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '79' - DOLocationID: '164' - payment_type: '1' - primary_passenger: - email: America72@gmail.com - mobile: 528-291-8014 x700 - fare_details: - fare_amount: '11.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '2.55' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '15.35' - congestion_surcharge: '' - - tripID: 21b8ea90-d3ab-4fe6-890e-2b8a911500e1 - VendorID: '1' - tpep_pickup_datetime: '2024-02-05 00:54:38' - tpep_dropoff_datetime: '2023-08-23 01:01:13' - passenger_count: '1' - trip_distance: '1.00' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '107' - DOLocationID: '170' - payment_type: '1' - primary_passenger: - email: Autumn_Kerluke@hotmail.com - mobile: 542.726.7058 - fare_details: - fare_amount: '6.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '8.8' - congestion_surcharge: '' - - tripID: 080446ad-cf54-429f-82e2-e54f4f8d4a9c - VendorID: '1' - tpep_pickup_datetime: '2023-07-16 00:00:58' - tpep_dropoff_datetime: '2023-10-29 00:06:38' - passenger_count: '1' - trip_distance: '.80' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '148' - DOLocationID: '79' - payment_type: '1' - primary_passenger: - email: Reanna_Conroy-Ratke@gmail.com - mobile: (565) 628-3638 - fare_details: - fare_amount: '5.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '2' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '8.8' - congestion_surcharge: '' - - tripID: 5ec8c804-9d6c-436b-be68-da3fa2ffcc8e - VendorID: '1' - tpep_pickup_datetime: '2023-03-08 00:14:58' - tpep_dropoff_datetime: '2024-02-09 00:24:33' - passenger_count: '1' - trip_distance: '2.70' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '4' - DOLocationID: '87' - payment_type: '1' - primary_passenger: - email: Randi0@yahoo.com - mobile: (248) 538-4300 x71383 - fare_details: - fare_amount: '11' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '13.3' - congestion_surcharge: '' - - tripID: 3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8 - VendorID: '1' - tpep_pickup_datetime: '2023-07-01 00:31:12' - tpep_dropoff_datetime: '2023-09-29 00:38:08' - passenger_count: '1' - trip_distance: '.70' - RatecodeID: '1' - store_and_fwd_flag: 'Y' - PULocationID: '148' - DOLocationID: '148' - payment_type: '1' - primary_passenger: - email: Ethyl74@yahoo.com - mobile: 292-960-2200 x317 - fare_details: - fare_amount: '6' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.8' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.1' - congestion_surcharge: '' - - tripID: dec9470a-6baa-492e-9220-d7ed626e2a99 - VendorID: '1' - tpep_pickup_datetime: '2024-02-22 00:43:21' - tpep_dropoff_datetime: '2024-03-02 00:50:49' - passenger_count: '1' - trip_distance: '1.10' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '79' - DOLocationID: '231' - payment_type: '1' - primary_passenger: - email: Makayla_Schneider18@hotmail.com - mobile: 1-885-537-0198 x47953 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.3' - congestion_surcharge: '' - - tripID: 904d9b1f-1f17-4cfc-8e63-8bf57257da5e - VendorID: '1' - tpep_pickup_datetime: '2023-12-29 00:54:14' - tpep_dropoff_datetime: '2023-05-03 01:02:32' - passenger_count: '1' - trip_distance: '1.50' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '231' - DOLocationID: '158' - payment_type: '1' - primary_passenger: - email: Cora.Grimes@yahoo.com - mobile: 843.706.7413 - fare_details: - fare_amount: '7.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.8' - congestion_surcharge: '' - - tripID: 57d8ed83-afa7-44cb-a0d4-723775602c34 - VendorID: '1' - tpep_pickup_datetime: '2024-02-02 00:16:34' - tpep_dropoff_datetime: '2023-09-11 00:24:45' - passenger_count: '2' - trip_distance: '1.20' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '142' - DOLocationID: '237' - payment_type: '2' - primary_passenger: - email: Tate.Bins@hotmail.com - mobile: (606) 682-9953 x671 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '8.3' - congestion_surcharge: '' - - tripID: fff2f2c8-4a30-446c-90af-443dd493886c - VendorID: '1' - tpep_pickup_datetime: '2023-12-03 00:29:04' - tpep_dropoff_datetime: '2023-07-26 00:36:33' - passenger_count: '2' - trip_distance: '1.90' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '237' - DOLocationID: '262' - payment_type: '1' - primary_passenger: - email: Magnus.Jacobs@hotmail.com - mobile: (955) 449-9284 x00149 - fare_details: - fare_amount: '8' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.85' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '11.15' - congestion_surcharge: '' - - tripID: 004c4cc7-e809-4108-aa01-87b0ded794ad - VendorID: '1' - tpep_pickup_datetime: '2023-05-18 00:41:00' - tpep_dropoff_datetime: '2023-08-05 00:59:50' - passenger_count: '1' - trip_distance: '5.80' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '236' - DOLocationID: '244' - payment_type: '2' - primary_passenger: - email: Palma24@gmail.com - mobile: 1-861-673-8247 x142 - fare_details: - fare_amount: '20' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '21.3' - congestion_surcharge: '' - - tripID: 1a01450d-537f-4f16-a2e9-f17021b077e9 - VendorID: '2' - tpep_pickup_datetime: '2024-01-31 00:20:53' - tpep_dropoff_datetime: '2023-12-21 00:40:21' - passenger_count: '2' - trip_distance: '5.70' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '164' - DOLocationID: '255' - payment_type: '2' - primary_passenger: - email: Jeffry18@gmail.com - mobile: (206) 748-5730 x64895 - fare_details: - fare_amount: '19.5' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '5.76' - improvement_surcharge: '0.3' - total_amount: '26.56' - congestion_surcharge: '' - - tripID: 5bccfd17-7813-43ca-9917-f2fe6ad49261 - VendorID: '2' - tpep_pickup_datetime: '2023-04-25 00:06:30' - tpep_dropoff_datetime: '2023-10-31 00:08:31' - passenger_count: '1' - trip_distance: '.46' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '239' - DOLocationID: '239' - payment_type: '1' - primary_passenger: - email: Leanne.Swaniawski@gmail.com - mobile: (787) 969-9302 x2684 - fare_details: - fare_amount: '4' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '1.59' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '6.89' - congestion_surcharge: '' - - tripID: e613b176-be28-4414-9a40-cff61e46db3b - VendorID: '2' - tpep_pickup_datetime: '2023-08-13 00:09:35' - tpep_dropoff_datetime: '2023-04-01 00:17:05' - passenger_count: '1' - trip_distance: '1.72' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '239' - DOLocationID: '236' - payment_type: '2' - primary_passenger: - email: Michel_Watsica63@yahoo.com - mobile: 793.751.7397 x892 - fare_details: - fare_amount: '8' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '9.3' - congestion_surcharge: '' - - tripID: 02e07922-e8a5-4655-84a8-b5ba1866f9fe - VendorID: '2' - tpep_pickup_datetime: '2023-04-28 00:18:42' - tpep_dropoff_datetime: '2024-02-15 00:24:38' - passenger_count: '1' - trip_distance: '1.60' - RatecodeID: '1' - store_and_fwd_flag: 'N' - PULocationID: '236' - DOLocationID: '239' - payment_type: '2' - primary_passenger: - email: Dewayne_Kuvalis17@gmail.com - mobile: 1-429-628-3797 x14211 - fare_details: - fare_amount: '7' - extra: '0.5' - mta_tax: '0.5' - tip_amount: '0' - tolls_amount: '0' - improvement_surcharge: '0.3' - total_amount: '8.3' - congestion_surcharge: '' - config: - dataset: generate-schema - responses: - '200': - description: OK - content: - application/json: - schema: - type: object - example: - id: dataset.schema.identify - ver: v1 - ts: 1721100502574 - params: - status: SUCCESS - errmsg: '' - responseCode: OK - result: - schema: - $schema: https://json-schema.org/draft/2020-12/schema - type: object - properties: - tripID: - type: string - suggestions: - - message: >- - The Property 'tripID' appears to be 'uuid' format - type. - advice: Suggest to not to index the high cardinal columns - resolutionType: DEDUP - severity: LOW - path: properties.tripID - arrival_format: text - data_type: string - VendorID: - type: string - arrival_format: text - data_type: string - tpep_pickup_datetime: - type: string - suggestions: - - message: >- - The Property 'tpep_pickup_datetime' appears to be - 'date-time' format type. - advice: The System can index all data on this column - resolutionType: INDEX - severity: LOW - path: properties.tpep_pickup_datetime - arrival_format: text - data_type: date-time - tpep_dropoff_datetime: - type: string - suggestions: - - message: >- - The Property 'tpep_dropoff_datetime' appears to be - 'date-time' format type. - advice: The System can index all data on this column - resolutionType: INDEX - severity: LOW - path: properties.tpep_dropoff_datetime - arrival_format: text - data_type: date-time - passenger_count: - type: string - arrival_format: text - data_type: string - trip_distance: - type: string - arrival_format: text - data_type: string - RatecodeID: - type: string - arrival_format: text - data_type: string - store_and_fwd_flag: - type: string - arrival_format: text - data_type: string - PULocationID: - type: string - arrival_format: text - data_type: string - DOLocationID: - type: string - arrival_format: text - data_type: string - payment_type: - type: string - arrival_format: text - data_type: string - primary_passenger: - type: object - properties: - email: - type: string - arrival_format: text - data_type: string - mobile: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - fare_details: - type: object - properties: - fare_amount: - type: string - arrival_format: text - data_type: string - extra: - type: string - arrival_format: text - data_type: string - mta_tax: - type: string - arrival_format: text - data_type: string - tip_amount: - type: string - arrival_format: text - data_type: string - tolls_amount: - type: string - arrival_format: text - data_type: string - improvement_surcharge: - type: string - arrival_format: text - data_type: string - total_amount: - type: string - arrival_format: text - data_type: string - congestion_surcharge: - type: string - arrival_format: text - data_type: string - arrival_format: object - data_type: object - additionalProperties: true - configurations: - indexConfiguration: - index: - Event Arrival Time: obsrv_meta.syncts - rollupSuggestions: - summary: - tripID: - path: $.tripID - cardinality: 98 - index: false - processing: - dedupKeys: - - tripID - dropDuplicates: - - 'Yes' - - 'No' - dataMappings: - text: - arrival_format: - - string - store_format: - string: - jsonSchema: string - datasource: string - date-time: - jsonSchema: string - datasource: string - date: - jsonSchema: string - datasource: string - boolean: - jsonSchema: string - datasource: boolean - epoch: - jsonSchema: string - datasource: integer - long: - jsonSchema: string - datasource: long - double: - jsonSchema: string - datasource: double - bigdecimal: - jsonSchema: string - datasource: double - integer: - jsonSchema: string - datasource: long - number: - arrival_format: - - number - - integer - store_format: - integer: - jsonSchema: integer - datasource: long - float: - jsonSchema: number - datasource: double - long: - jsonSchema: integer - datasource: long - double: - jsonSchema: number - datasource: double - bigdecimal: - jsonSchema: number - datasource: double - epoch: - jsonSchema: integer - datasource: long - number: - jsonSchema: number - datasource: double - object: - arrival_format: - - object - store_format: - object: - jsonSchema: object - datasource: json - array: - arrival_format: - - array - store_format: - array: - jsonSchema: array - datasource: array - boolean: - arrival_format: - - boolean - store_format: - boolean: - jsonSchema: boolean - datasource: boolean /v2/datasets/update: patch: tags: @@ -5884,4 +4010,135 @@ paths: code: DATA_SCHEMA_INVALID_INPUT message: >- #properties/request/required must have required property - 'config' \ No newline at end of file + 'config' + /v2/files/generate-url: + post: + tags: + - Dataset api's + summary: File generate url + requestBody: + content: + application/json: + examples: + example1: + summary: Request body for put url + value: + id: api.files.generate-url + ver: v1 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: write + example2: + summary: Request body for get url + value: + id: api.files.generate-url + ver: v1 + ts: '2024-04-19T12:58:47+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + request: + files: + - telemetry.json + - school_data.json + access: read + parameters: + - name: Content-Type + in: header + schema: + type: string + example: application/json + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Generate put url' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T08:26:19+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 5306f309-4a15-458e-89e2-29d8ac0835d4 + responseCode: OK + result: + - filePath: >- + test-connector/api-service/user_uploads/telemetry_10d595.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject + - filePath: >- + test-connector/api-service/user_uploads/school_data_33109a.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject + example-1: + summary: 'Success: Generate get url' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T09:31:40+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: 009c0b2d-8acd-40b0-a807-bbacf9242771 + responseCode: OK + result: + - filePath: test-connector/api-service/user_uploads/telemetry.json + fileName: telemetry.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject + - filePath: >- + test-connector/api-service/user_uploads/school_data.json + fileName: school_data.json + preSignedUrl: >- + https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject + '400': + description: Bad Request + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Failure: limit exceeds' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T08:33:04+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: d3a606ca-47d0-4746-95a1-c8692e749959 + responseCode: BAD_REQUEST + error: + code: FILES_URL_GENERATION_LIMIT_EXCEED + message: 'Pre-signed URL generation failed: limit exceeded.' + trace: '' + example-1: + summary: 'Failure: Invalid request' + value: + id: api.files.generate-url + ver: v1 + ts: '2024-07-16T09:31:10+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6 + resmsgid: c3e9da1c-09f3-4a3b-84ec-a19efc68b856 + responseCode: BAD_REQUEST + error: + code: FILES_GENERATE_URL_INPUT_INVALID + message: >- + #properties/request/properties/access/enum must be equal + to one of the allowed values + trace: '' \ No newline at end of file From 636c7dd70dde506b71aa3236e11b72c98730fa87 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 22 Jul 2024 15:17:25 +0530 Subject: [PATCH 046/235] #OBS-I126 : updated server url --- api-service/swagger-doc/v2_updated_doc_openapi.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml index f51caf14..ea2a663c 100644 --- a/api-service/swagger-doc/v2_updated_doc_openapi.yml +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -3,8 +3,7 @@ info: title: Obsrv api documentation version: 1.0.0 servers: - - url: localhost:3000 - - url: http://localhost:3007 + - url: http://localhost:3000 tags: - name: Dataset api's paths: From c8ef5e15d1a02df5515420a9430e6078d61f1721 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Mon, 22 Jul 2024 16:47:17 +0530 Subject: [PATCH 047/235] #OBS-I21: feat: dataset publish changes for connectors --- .../spark-connector-cron/values.yaml | 4 +- .../src/command/connector_command.py | 86 ++++++++++++------- .../src/command/connector_registry.py | 21 ++++- command-service/src/config/service_config.yml | 10 +++ command-service/src/model/data_models.py | 2 +- command-service/src/model/db_models.py | 9 ++ 6 files changed, 95 insertions(+), 37 deletions(-) diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml index af90fa31..a781b613 100644 --- a/command-service/helm-charts/spark-connector-cron/values.yaml +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -1,5 +1,5 @@ nameOverride: "" -fullnameOverride: "spark-submit-cron" +fullnameOverride: "" namespace: "spark" replicaCount: 1 @@ -162,5 +162,5 @@ main_file: jdbc-connector-1.0.0.jar # main_file: object_store_connector/__main__.py # The schedule in cron format -cronSchedule: 0 * * * * # Every hour +cronSchedule: 0 0 0 * * # Every month restartPolicy: OnFailure \ No newline at end of file diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 17d2d603..e5f67970 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -1,9 +1,12 @@ +from dacite import from_dict +import json import logging import subprocess from command.icommand import ICommand from config import Config from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType +from model.db_models import ConnectorInstance from service.db_service import DatabaseService @@ -12,10 +15,7 @@ def __init__(self, config: Config, db_service: DatabaseService): self.config = config self.db_service = db_service self.logger = logging.getLogger() - self.connector_job_ns = "connector-jobs" - self.connector_job_chart_dir = "{0}/connector-cron-jobs".format( - self.config.find("helm_charts_base_dir") - ) + self.connector_job_config = self.config.find("connector_jobs") def execute(self, command_payload: CommandPayload, action: Action): result = None @@ -25,6 +25,10 @@ def execute(self, command_payload: CommandPayload, action: Action): is_masterdata = self._get_masterdata_details( dataset_id=command_payload.dataset_id ) + + print(f"Active connectors: {active_connectors}") + print(f"Is masterdata: {is_masterdata}") + if action == Action.DEPLOY_CONNECTORS.name: result = self._deploy_connectors( command_payload.dataset_id, active_connectors, is_masterdata @@ -34,7 +38,7 @@ def execute(self, command_payload: CommandPayload, action: Action): def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): result = None - self._stop_connector_jobs(dataset_id, active_connectors, is_masterdata) + # self._stop_connector_jobs(dataset_id, active_connectors, is_masterdata) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) return result @@ -82,42 +86,59 @@ def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata): def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None + for connector in active_connectors: print("Installing connector {0}".format(connector)) - connector_jar_config = self.config.find("connector_job")[connector] - for release in connector_jar_config: - result = self._perform_install(release) - if is_masterdata: - print("Installing masterdata job") - masterdata_jar_config = self.config.find("masterdata_job") - for release in masterdata_jar_config: - result = self._perform_install(release) + + if connector.connector_runtime == "spark": + result = self._perform_spark_install(connector) + else: + print( + f"Connector {connector.connector_id} is not supported for deployment" + ) + break + + # if is_masterdata: + # print("Installing masterdata job") + # masterdata_jar_config = self.config.find("masterdata_job") + # for release in masterdata_jar_config: + # result = self._perform_install(release) return result - def _perform_install(self, release): + def _perform_spark_install(self, connector_instance): err = None result = None - release_name = release["release_name"] + release_name = connector_instance.id + # print("Instance -->>", connector_instance) + connector_source = json.loads(connector_instance.connector_source) + print(connector_source) helm_install_cmd = [ "helm", "upgrade", "--install", release_name, - self.connector_job_chart_dir, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", "--namespace", - self.connector_job_ns, + self.connector_job_config["spark"]["namespace"], "--create-namespace", "--set", - "file.path={}".format(release["jar"]), + "technology={}".format(connector_instance.technology), + "--set", + "instance_id={}".format(release_name), + "--set", + "connector_source={}".format(connector_source["source"]), "--set", - "class.name={}".format(release["class"]), + "main_class={}".format(connector_source["main_class"]), "--set", - "job.name={}".format(release_name), + "main_file={}".format(connector_source["main_program"]), "--set", - "args={}".format(",".join(release["args"])), + "serviceAccount.create=false", "--set", - "schedule={}".format(release["schedule"]), + "cronSchedule={}".format(connector_instance.operations_config["schedule"]) ] + + print(" ".join(helm_install_cmd)) + helm_install_result = subprocess.run( helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @@ -128,7 +149,7 @@ def _perform_install(self, release): result = ActionResponse( status="ERROR", status_code=500, - error_message="FLINK_HELM_INSTALLATION_EXCEPTION", + error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", ) print( f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" @@ -141,15 +162,19 @@ def _perform_install(self, release): def _get_connector_details(self, dataset_id): active_connectors = [] - rows = self.db_service.execute_select_all( + records = self.db_service.execute_select_all( f""" - SELECT DISTINCT(connector_type) - FROM dataset_source_config - WHERE status='{DatasetStatusType.Live.name}' and connector_type IN ('object', 'jdbc')""" + SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology + FROM connector_instances ci + JOIN connector_registry cr on ci.connector_id = cr.id + WHERE ci.status='{DatasetStatusType.Live.name}' and ci.dataset_id = '{dataset_id}' + """ ) - for row in rows: - active_connectors.append(row[0]) + for record in records: + active_connectors.append(from_dict( + data_class=ConnectorInstance, data=record + )) return active_connectors @@ -159,7 +184,8 @@ def _get_masterdata_details(self, dataset_id): f""" SELECT * FROM datasets - WHERE status='{DatasetStatusType.Live.name}' AND dataset_id = '{dataset_id}' AND type = 'master-dataset'""" + WHERE status='{DatasetStatusType.Live.name}' AND dataset_id = '{dataset_id}' AND type = 'master' + """ ) if len(rows) > 0: diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index cef727c6..36cb0b57 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -177,6 +177,12 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: if tenant == "multiple": connector_objects = self.metadata["connectors"] for obj in connector_objects: + source = { + "source": connector_source, + "main_class": obj["main_class"] if "main_class" in obj else self.metadata["metadata"]["main_class"], + "main_program": obj["main_program"] if "main_program" in obj else self.metadata["metadata"]["main_program"], + } + connector_id = obj["id"].replace(" ", "-") registry_meta = ConnectorRegsitryv2(connector_id, obj['name'], @@ -191,7 +197,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: obj['icon'], 'Live', rel_path, - connector_source, + json.dumps(source), 'SYSTEM', datetime.now().strftime("%Y-%m-%d %H:%M:%S"), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), @@ -219,6 +225,13 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: connector_id = ( self.metadata.get("metadata", {}).get("id", "").replace(" ", "-") ) + + source = { + "source": connector_source, + "main_class": self.metadata["metadata"]["main_class"], + "main_program": self.metadata["metadata"]["main_program"], + } + registry_meta = ConnectorRegsitryv2( connector_id, self.metadata['metadata']['name'], @@ -233,11 +246,11 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: self.metadata['metadata']['icon'], 'Live', rel_path, - connector_source, + json.dumps(source), 'SYSTEM', datetime.now().strftime("%Y-%m-%d %H:%M:%S"), datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - self.ui_spec[obj["id"]] if obj["id"] in self.ui_spec else {}, + self.ui_spec, 'SYSTEM', datetime.now().strftime("%Y-%m-%d %H:%M:%S") ) @@ -252,7 +265,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: return RegistryResponse( status="success", message="Connectors registered successfully", - connector_info=registry_meta, + connector_info=[registry_meta.to_dict()], statusCode=status.HTTP_200_OK, ) diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index c580ed14..a26d4a53 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -24,6 +24,9 @@ commands: RESTART_PIPELINE: workflow: - START_PIPELINE_JOBS + RESTART_CONNECTORS: + workflow: + - DEPLOY_CONNECTORS alert_manager: metrics: @@ -164,6 +167,13 @@ kafka: telemetry: topic: system.telemetry.events +connector_jobs: + spark: + namespace: spark + base_helm_chart: spark-connector-cron + flink: + namespace: streaming-connectors + base_helm_chart: flink-connector connector_registry: download_path: /tmp/connector-registry diff --git a/command-service/src/model/data_models.py b/command-service/src/model/data_models.py index 198d649b..f248e6ba 100644 --- a/command-service/src/model/data_models.py +++ b/command-service/src/model/data_models.py @@ -9,7 +9,7 @@ class Command(Enum): PUBLISH_DATASET = "PUBLISH_DATASET" RESTART_PIPELINE = "RESTART_PIPELINE" - + RESTART_CONNECTORS = "RESTART_CONNECTORS" class Action(Enum): SUBMIT_INGESTION_TASKS = "SUBMIT_INGESTION_TASKS" diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index 1999e8b5..a8c12c8d 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -125,3 +125,12 @@ class ConnectorRegsitryv2: ui_spec: dict | None = None updated_by: str | None = None livedate: datetime | None = None + +@dataclass +class ConnectorInstance: + id: str + connector_id: str + operations_config: dict + connector_runtime: str + connector_source: str + technology: str From 761c8379c6a4ff3e70c02475e0e018d8b4753857 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 24 Jul 2024 12:30:14 +0530 Subject: [PATCH 048/235] #OBS-I1 updated Dataset Health API code --- api-service/package.json | 2 +- .../DatasetHealth/DatasetHealth.ts | 140 ++++-------------- api-service/src/v2/services/DatasetService.ts | 4 + api-service/src/v2/services/HealthService.ts | 48 +++++- api-service/src/v2/types/DatasetModels.ts | 4 + 5 files changed, 82 insertions(+), 116 deletions(-) diff --git a/api-service/package.json b/api-service/package.json index 709b3f4b..f760df95 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -47,7 +47,7 @@ "pg": "^8.11.3", "pg-hstore": "^2.3.4", "prom-client": "^14.2.0", - "redis": "^4.6.14", + "redis": "^4.6.15", "sequelize": "^6.37.1", "slug": "^9.0.0", "trino-client": "^0.2.2", diff --git a/api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts b/api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts index ad09e84c..610bfbc3 100644 --- a/api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts +++ b/api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts @@ -2,124 +2,48 @@ import { Request, Response } from "express"; import _ from "lodash"; import { schemaValidation } from "../../services/ValidationService"; import DatasetHealthRequestSchema from "./DatasetHealthValidationSchema.json" -import { ErrorObject } from "../../types/ResponseModel"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { DatasetStatus, DatasetType, HealthStatus } from "../../types/DatasetModels"; -import { Dataset } from "../../models/Dataset"; -import logger from "../../logger"; -import { getInfraHealth, getProcessingHealth, getQueryHealth } from "../../services/HealthService"; -import { Datasource } from "../../models/Datasource"; - +import { DatasetStatus } from "../../types/DatasetModels"; +import { getDatasetHealth, getInfraHealth } from "../../services/HealthService"; +import { obsrvError } from "../../types/ObsrvError"; +import httpStatus from "http-status"; +import { datasetService } from "../../services/DatasetService"; export const apiId = "api.dataset.health"; -export const errorCode = "DATASET_HEALTH_FAILURE" - +const validateRequest = async (req: Request) => { + const isRequestValid: Record = schemaValidation(req.body, DatasetHealthRequestSchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASET_HEALTH_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) + } +} const datasetHealth = async (req: Request, res: Response) => { - const resmsgid = _.get(res, "resmsgid"); - const requestBody = req.body; - const msgid = _.get(req, ["body", "params", "msgid"]); - try { - const isRequestValid: Record = schemaValidation(req.body, DatasetHealthRequestSchema) - if (!isRequestValid.isValid) { - const code = "DATASET_HEALTH_INPUT_INVALID" - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - if (requestBody?.request?.dataset === "*") { - const {components, status} = await getInfraHealth(false) - return ResponseHandler.successResponse(req, res, { - status: 200, data: { - "status": status, - "details": [ - { - "category": "infra", - "status": status, - "components": components - }] - } - }); - } - - const dataset = await getLiveDatasets(requestBody?.request?.dataset) - if(_.isEmpty(dataset)) { - const code = "DATASET_HEALTH_NO_DATASETS" - const message = `There are no live dataset exists with given dataset_id: ${requestBody?.request?.dataset}` - logger.error({ code, apiId, msgid, requestBody, resmsgid, message: message }) - return ResponseHandler.errorResponse({ - code, - message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - logger.debug(apiId, msgid, resmsgid, "dataset", dataset) - - const categories = requestBody?.request?.categories - const details = [] - if(requestBody?.request?.categories.includes("infra")) { - const isMasterDataset = _.get(dataset, "[0].type") == DatasetType.MasterDataset; - logger.debug({isMasterDataset}) - const {components, status} = await getInfraHealth(isMasterDataset) - details.push({ - "category": "infra", - "status": status, - "components": components - }) - } - logger.debug({categories}) - if(categories.includes("processing")) { - const {components, status} = await getProcessingHealth(dataset[0]) - details.push({ - "category": "processing", - "status": status, - "components": components - }) - } - - if(categories.includes("query")) { - const datasources = await getDataSources(requestBody?.request?.dataset) - const {components, status} = await getQueryHealth(datasources, dataset[0]) - details.push({ - "category": "query", - "status": status, - "components": components - }) - } - - const allStatus = _.includes(_.map(details, (detail) => detail?.status), HealthStatus.UnHealthy) ? HealthStatus.UnHealthy: HealthStatus.Healthy + const dataset_id = _.get(req, ["body", "request", "dataset_id"]); + const categories = _.get(req, ["body", "request", "categories"]) + validateRequest(req) + if (dataset_id === "*") { + const { components, status } = await getInfraHealth(false) return ResponseHandler.successResponse(req, res, { - status: 200, data: { - "status": allStatus, - "details": details + status: httpStatus.OK, data: { + status, + details: [ + { + category: "infra", + status, + components + }] } }); - - } catch (error: any) { - logger.error({ ...error, apiId, code: errorCode, msgid, requestBody, resmsgid }); - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: errorCode, message: "Failed to check dataset health" } - } - ResponseHandler.errorResponse(errorMessage, req, res); } - - - -} -const getLiveDatasets = async (ids: Record): Promise> => { - return Dataset.findAll({ attributes: ["dataset_id", "status", "type"], where: { dataset_id: ids, status: DatasetStatus.Live }, raw: true }); -} - -const getDataSources = async (ids: Record): Promise> => { - return Datasource.findAll({ attributes: ["dataset_id", "datasource"], where: { dataset_id: ids }, raw: true }); + const dataset = await datasetService.findDatasets({ id: dataset_id, status: DatasetStatus.Live }, ["dataset_id", "status", "type"]) + if (_.isEmpty(dataset)) { + throw obsrvError(dataset, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset} not found`, "NOT_FOUND", 404); + } + const datasetHealthData = await getDatasetHealth(categories, dataset[0]) + return ResponseHandler.successResponse(req, res, { + status: httpStatus.OK, data: datasetHealthData + }); } export default datasetHealth; \ No newline at end of file diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 43bbcdf0..b86d5a27 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -225,6 +225,10 @@ class DatasetService { } } + findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { + return Datasource.findAll({where, attributes, order, raw: true}) + } + private deleteDruidSupervisors = async (dataset: Record) => { try { diff --git a/api-service/src/v2/services/HealthService.ts b/api-service/src/v2/services/HealthService.ts index 0f9244ca..16ece7dc 100644 --- a/api-service/src/v2/services/HealthService.ts +++ b/api-service/src/v2/services/HealthService.ts @@ -2,15 +2,14 @@ import axios from "axios"; import { config } from "../configs/Config"; import { logger } from "@project-sunbird/logger"; import { health as postgresHealth } from "../connections/databaseConnection"; -import { DatasetType, HealthStatus } from "../types/DatasetModels"; +import { DatasetType, DataSourceType, HealthStatus } from "../types/DatasetModels"; import { createClient } from "redis"; import { isHealthy as isKafkaHealthy } from "../connections/kafkaConnection"; import { druidHttpService, executeNativeQuery } from "../connections/druidConnection"; import _ from "lodash"; import moment from "moment"; import { SystemConfig } from "./SystemConfig"; - - +import { datasetService } from "./DatasetService"; const dateFormat = "YYYY-MM-DDT00:00:00+05:30" const prometheusInstance = axios.create({ baseURL: config?.query_api?.prometheus?.url, headers: { "Content-Type": "application/json" } }); @@ -20,7 +19,7 @@ const init = async () => { createClient({ url: `redis://${config.redis_config.denorm_redis_host}:${config.redis_config.denorm_redis_port}` }) - .on("error", err => { + .on("error", (err: any) => { logger.error("unable to connect to denorm redis client", err) isRedisDenormHealthy = false }) @@ -35,13 +34,50 @@ const init = async () => { .on("ready", () => { isRedisDedupHealthy = true }) - .on("error", err => { + .on("error", (err: any) => { isRedisDedupHealthy = false logger.error("unable to connect to dedup redis client", err) }) .connect(); } +export const getDatasetHealth = async (categories: any, dataset: any) => { + + const details = [] + if (categories.includes("infra")) { + const isMasterDataset = _.get(dataset, "[0].type") == DatasetType.master; + const { components, status } = await getInfraHealth(isMasterDataset) + details.push({ + "category": "infra", + "status": status, + "components": components + }) + } + if (categories.includes("processing")) { + const { components, status } = await getProcessingHealth(dataset[0]) + details.push({ + "category": "processing", + "status": status, + "components": components + }) + } + + if (categories.includes("query")) { + const datasources = await datasetService.findDatasources({ dataset_id: dataset.id, type: DataSourceType.druid }, ["dataset_id", "datasource"]) + const { components, status } = await getQueryHealth(datasources, dataset[0]) + details.push({ + "category": "query", + "status": status, + "components": components + }) + } + + const allStatus = _.includes(_.map(details, (detail) => detail?.status), HealthStatus.UnHealthy) ? HealthStatus.UnHealthy : HealthStatus.Healthy + return { + "status": allStatus, + "details": details + } +} const getDatasetIdForMetrics = (datasetId: string) => { datasetId = datasetId.replace(/-/g, "_") .replace(/\./g, "_") @@ -516,6 +552,4 @@ const getQueriesFailedCount = async (datasetId: string) => { } } - - init().catch(err => logger.error(err)) \ No newline at end of file diff --git a/api-service/src/v2/types/DatasetModels.ts b/api-service/src/v2/types/DatasetModels.ts index 0e933abc..e67f902b 100644 --- a/api-service/src/v2/types/DatasetModels.ts +++ b/api-service/src/v2/types/DatasetModels.ts @@ -66,4 +66,8 @@ export enum DatasetAction { export enum HealthStatus { Healthy = "Healthy", UnHealthy= "UnHealthy" +} + +export enum DataSourceType { + druid = "druid" } \ No newline at end of file From 2b3c57bace7485825d22fe4da9a88cb4c32a144c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 24 Jul 2024 14:58:44 +0530 Subject: [PATCH 049/235] #OBS-I116: Dataset CRUD api fixes --- .../DatasetCreateValidationSchema.json | 64 +++++++++++++++++-- .../v2/controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition.ts | 12 ++-- .../DatasetUpdateValidationSchema.json | 62 ++++++++++++++++-- .../src/v2/models/ConnectorInstances.ts | 3 +- .../src/v2/models/ConnectorRegistry.ts | 3 +- api-service/src/v2/services/DatasetService.ts | 27 ++++---- 7 files changed, 138 insertions(+), 35 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index 938fca6d..72fa1926 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -49,8 +49,18 @@ "default": "Strict" } }, - "required": ["validate", "mode"], - "additionalProperties": false + "required": ["validate"], + "additionalProperties": false, + "if": { + "properties": { + "validate": { + "const": true + } + } + }, + "then": { + "required": ["mode"] + } }, "extraction_config": { "type": "object", @@ -76,12 +86,37 @@ "minLength": 1 } }, - "required": ["drop_duplicates", "dedup_key"], + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], "additionalProperties": false } }, - "required": ["is_batch_event", "extraction_key", "dedup_config"], - "additionalProperties": false + "additionalProperties": false, + "required": ["is_batch_event"], + "if": { + "properties": { + "is_batch_event": { + "const": true + } + } + }, + "then": { + "required": ["extraction_key", "dedup_config"] + } }, "dedup_config": { "type": "object", @@ -95,7 +130,22 @@ "minLength": 1 } }, - "required": ["drop_duplicates", "dedup_key"], + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], "additionalProperties": false }, "data_schema": { @@ -297,7 +347,7 @@ "type": "array", "items": { "type": "string", - "minLength":1 + "minLength": 1 } }, "sample_data": { diff --git a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts index c82008fc..610deca8 100644 --- a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts @@ -43,7 +43,7 @@ const datasetRead = async (req: Request, res: Response) => { const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { - const attrs = _.union(attributes, ["dataset_config", "api_version", "type"]) + const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); if(draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 948a7772..aed0f9e4 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -145,12 +145,14 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) dataset_id: denormField.dataset_id, exists: (md) ? true : false, isLive: (md) ? md.status === "Live" : false, - status: md.status + status: (md) ? md.status : false + } + if(!_.isEmpty(md)){ + if(md.api_version === "v2") + datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); + else + datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); } - if(md.api_version === "v2") - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); - else - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); return datasetStatus; }) diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 929ac0bb..76f50543 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -47,8 +47,18 @@ "enum": ["Strict", "IgnoreNewFields"] } }, - "required": ["validate", "mode"], - "additionalProperties": false + "required": ["validate"], + "additionalProperties": false, + "if": { + "properties": { + "validate": { + "const": true + } + } + }, + "then": { + "required": ["mode"] + } }, "extraction_config": { "type": "object", @@ -72,12 +82,37 @@ "type": "string" } }, - "required": ["drop_duplicates", "dedup_key"], + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], "additionalProperties": false } }, - "required": ["is_batch_event", "extraction_key", "dedup_config"], - "additionalProperties": false + "additionalProperties": false, + "required": ["is_batch_event"], + "if": { + "properties": { + "is_batch_event": { + "const": true + } + } + }, + "then": { + "required": ["extraction_key", "dedup_config"] + } }, "dedup_config": { "type": "object", @@ -91,7 +126,22 @@ "minLength": 1 } }, - "required": ["drop_duplicates", "dedup_key"], + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], "additionalProperties": false }, "data_schema": { diff --git a/api-service/src/v2/models/ConnectorInstances.ts b/api-service/src/v2/models/ConnectorInstances.ts index df840042..14c832d7 100644 --- a/api-service/src/v2/models/ConnectorInstances.ts +++ b/api-service/src/v2/models/ConnectorInstances.ts @@ -49,6 +49,5 @@ export const ConnectorInstances = sequelize.define("connector_instances", { tableName: "connector_instances", timestamps: true, createdAt: "created_date", - updatedAt: "updated_date", - paranoid: true + updatedAt: "updated_date" }) \ No newline at end of file diff --git a/api-service/src/v2/models/ConnectorRegistry.ts b/api-service/src/v2/models/ConnectorRegistry.ts index 64ecad46..b31e5b27 100644 --- a/api-service/src/v2/models/ConnectorRegistry.ts +++ b/api-service/src/v2/models/ConnectorRegistry.ts @@ -81,6 +81,5 @@ export const ConnectorRegistry = sequelize.define("connector_registry", { tableName: "connector_registry", timestamps: true, createdAt: "created_date", - updatedAt: "updated_date", - paranoid: true + updatedAt: "updated_date" }) \ No newline at end of file diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 43bbcdf0..514641f1 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -92,18 +92,20 @@ class DatasetService { return responseData; } - migrateDraftDataset = async (datasetId: string, dataset: Model): Promise => { + migrateDraftDataset = async (datasetId: string, dataset: Record): Promise => { - let draftDataset : Record = { - api_version: "v2" + let draftDataset: Record = { + api_version: "v2", + version_key: Date.now().toString() } - const dataset_config:any = _.get(dataset, "dataset_config"); + const dataset_id = _.get(dataset, "id") + const dataset_config: any = _.get(dataset, "dataset_config"); draftDataset["dataset_config"] = { indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} } - const transformations = await this.getDraftTransformations(datasetId, ["field_key", "transformation_function", "mode", "metadata"]); + const transformations = await this.getDraftTransformations(dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); draftDataset["transformations_config"] = _.map(transformations, (config) => { return { field_key: _.get(config, ["field_key"]), @@ -113,7 +115,7 @@ class DatasetService { category: this.getTransformationCategory(_.get(config, ["metadata.section"])) } }) - const connectors = await this.getDraftConnectors(datasetId, ["connector_type", "connector_config"]); + const connectors = await this.getDraftConnectors(dataset_id, ["connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { return { connector_id: _.get(config, ["connector_type"]), @@ -124,9 +126,9 @@ class DatasetService { const transaction = await sequelize.transaction(); try { - await DatasetDraft.update(draftDataset, { where: { id: datasetId }, transaction}); - await DatasetTransformationsDraft.destroy({ where: { dataset_id: datasetId }, transaction }); - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: datasetId }, transaction }); + await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); + await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); + await DatasetSourceConfigDraft.destroy({ where: { dataset_id }, transaction }); await transaction.commit(); } catch (err) { await transaction.rollback(); @@ -166,7 +168,7 @@ class DatasetService { version: "v1" } }) - const transformations = await this.getDraftTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); + const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); draftDataset["transformations_config"] = _.map(transformations, (config) => { return { field_key: _.get(config, "field_key"), @@ -180,11 +182,12 @@ class DatasetService { } else { const connectors = await this.getConnectors(draftDataset.dataset_id, ["connector_id", "connector_config", "operations_config"]); draftDataset["connectors_config"] = connectors - const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "datatype", "category"]); + const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode"]); draftDataset["transformations_config"] = transformations } draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version + draftDataset["status"] = DatasetStatus.Draft await DatasetDraft.create(draftDataset); return await this.getDraftDataset(draftDataset.dataset_id); } @@ -259,11 +262,11 @@ class DatasetService { } } await transaction.commit() - await executeCommand(draftDataset.id, "PUBLISH_DATASET"); } catch(err:any) { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } + await executeCommand(draftDataset.id, "PUBLISH_DATASET"); } From 71140311c91cfbafade8884ff8897b73d0e56c5d Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 24 Jul 2024 15:30:03 +0530 Subject: [PATCH 050/235] #OBS-I116: fix: error codes fix --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index aed0f9e4..2d8d9ffb 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -38,11 +38,11 @@ const validateDataset = (dataset: any, datasetId: any, action: string) => { } if (dataset.api_version !== "v2" && _.includes(["ReadyToPublish", "Live"], action)) { - throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "NOT_FOUND", 404) + throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "BAD_REQUEST", 400) } if(!_.includes(allowedTransitions[action], dataset.status)) { - throw obsrvError(datasetId, `DATASET_${_.toUpper(action)}_FAILURE`, `Transition failed for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, "NOT_FOUND", 404) + throw obsrvError(datasetId, `DATASET_${_.toUpper(action)}_FAILURE`, `Transition failed for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, "CONFLICT", 409) } return true; @@ -134,7 +134,7 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) throw { code: "SELF_REFERENCING_MASTER_DATA", message: `The denorm master dataset is self-referencing itself`, - errCode: "SELF_REFERENCING_MASTER_DATA", + errCode: "CONFLICT", statusCode: 409 } } @@ -162,7 +162,7 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) throw { code: "DEPENDENT_MASTER_DATA_NOT_LIVE", message: `The datasets with id:${invalidIds} are not in published status`, - errCode: "DEPENDENT_MASTER_DATA_NOT_LIVE", + errCode: "PRECONDITION_REQUIRED", statusCode: 428 } } @@ -184,7 +184,7 @@ const updateMasterDataConfig = async (draftDataset: Record) => { throw { code: "REDIS_DB_INDEX_FETCH_FAILED", message: `Unable to fetch the redis db index for the master data`, - errCode: "REDIS_DB_INDEX_FETCH_FAILED", + errCode: "INTERNAL_SERVER_ERROR", statusCode: 500 } } From 3da2fd42e227d09d1e67bc24f76719ff9ad126b7 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 24 Jul 2024 15:56:24 +0530 Subject: [PATCH 051/235] #OBS-I1 Refactored as per new changes --- .../DatasetHealthValidationSchema.json | 4 +- api-service/src/v2/services/HealthService.ts | 192 ++++++------------ 2 files changed, 63 insertions(+), 133 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json b/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json index 5a788259..82c5a3ee 100644 --- a/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json @@ -25,7 +25,7 @@ "request": { "type": "object", "properties": { - "dataset": { + "dataset_id": { "type": "string", "minLength": 1 }, @@ -46,7 +46,7 @@ } }, "required": [ - "categories" + "categories", "dataset_id" ] } } diff --git a/api-service/src/v2/services/HealthService.ts b/api-service/src/v2/services/HealthService.ts index 16ece7dc..270496fe 100644 --- a/api-service/src/v2/services/HealthService.ts +++ b/api-service/src/v2/services/HealthService.ts @@ -12,40 +12,23 @@ import { SystemConfig } from "./SystemConfig"; import { datasetService } from "./DatasetService"; const dateFormat = "YYYY-MM-DDT00:00:00+05:30" -const prometheusInstance = axios.create({ baseURL: config?.query_api?.prometheus?.url, headers: { "Content-Type": "application/json" } }); -let isRedisDenormHealthy = false; -let isRedisDedupHealthy = false; -const init = async () => { - createClient({ - url: `redis://${config.redis_config.denorm_redis_host}:${config.redis_config.denorm_redis_port}` - }) - .on("error", (err: any) => { - logger.error("unable to connect to denorm redis client", err) - isRedisDenormHealthy = false - }) - .on("ready", () => { - isRedisDenormHealthy = true - }) - .connect(); - - createClient({ - url: `redis://${config.redis_config.dedup_redis_host}:${config.redis_config.dedup_redis_port}` - }) - .on("ready", () => { - isRedisDedupHealthy = true - }) - .on("error", (err: any) => { - isRedisDedupHealthy = false - logger.error("unable to connect to dedup redis client", err) - }) - .connect(); +const prometheusInstance = axios.create({ baseURL: config?.query_api?.prometheus?.url, headers: { "Content-Type": "application/json" } }) + +const prometheusQueries = { + validationFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_DATASETID_validator_failed_count[1d]))", + dedupFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_DATASETID_dedup_failed_count[1d]))", + denormFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_DenormalizerJob_DATASETID_denorm_failed[1d]))", + transformationFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_TransformerJob_DATASETID_transform_failed_count[1d]))", + queriesCount: 'sum(sum_over_time(node_total_api_calls{entity="data-out", dataset_id="DATASETID"}[1d]))', + avgQueryResponseTimeInSec: 'avg(avg_over_time(node_query_response_time{entity="data-out", dataset_id="DATASETID"}[1d]))/1000', + queriesFailedCount: 'sum(sum_over_time(node_failed_api_calls{entity="data-out", dataset_id="DATASETID"}[1d]))' } export const getDatasetHealth = async (categories: any, dataset: any) => { const details = [] if (categories.includes("infra")) { - const isMasterDataset = _.get(dataset, "[0].type") == DatasetType.master; + const isMasterDataset = _.get(dataset, "type") == DatasetType.master; const { components, status } = await getInfraHealth(isMasterDataset) details.push({ "category": "infra", @@ -54,7 +37,7 @@ export const getDatasetHealth = async (categories: any, dataset: any) => { }) } if (categories.includes("processing")) { - const { components, status } = await getProcessingHealth(dataset[0]) + const { components, status } = await getProcessingHealth(dataset) details.push({ "category": "processing", "status": status, @@ -64,7 +47,7 @@ export const getDatasetHealth = async (categories: any, dataset: any) => { if (categories.includes("query")) { const datasources = await datasetService.findDatasources({ dataset_id: dataset.id, type: DataSourceType.druid }, ["dataset_id", "datasource"]) - const { components, status } = await getQueryHealth(datasources, dataset[0]) + const { components, status } = await getQueryHealth(datasources, dataset) details.push({ "category": "query", "status": status, @@ -86,8 +69,16 @@ const getDatasetIdForMetrics = (datasetId: string) => { return datasetId; } -const queryMetrics = (params: Record | string) => { - return prometheusInstance.get("/api/v1/query", { params }) +const queryMetrics = async (datasetId: string, query: string) => { + const queryWithDatasetId = query.replace("DATASETID", getDatasetIdForMetrics(datasetId)) + try { + const { data } = await prometheusInstance.get("/api/v1/query", { params: {query: queryWithDatasetId} }) + return { count: _.toInteger(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } + } catch (error) { + logger.error(error) + return { count: 0, health: HealthStatus.UnHealthy } + } + } export const getInfraHealth = async (isMasterDataset: boolean): Promise<{ components: any, status: string }> => { @@ -103,7 +94,7 @@ export const getInfraHealth = async (isMasterDataset: boolean): Promise<{ compon { "type": "flink", "status": flink } ] if (isMasterDataset) { - redis = await getRedisStatus() + redis = await getRedisHealthStatus() components.push({ "type": "redis", "status": redis }) } const status = [postgres, redis, kafka, druid, flink].includes(HealthStatus.UnHealthy) ? HealthStatus.UnHealthy : HealthStatus.Healthy @@ -112,7 +103,7 @@ export const getInfraHealth = async (isMasterDataset: boolean): Promise<{ compon export const getProcessingHealth = async (dataset: any): Promise<{ components: any, status: string }> => { const dataset_id = _.get(dataset, "dataset_id") - const isMasterDataset = _.get(dataset, "type") == DatasetType.MasterDataset; + const isMasterDataset = _.get(dataset, "type") == DatasetType.master; const flink = await getFlinkHealthStaus() const { count, health } = await getEventsProcessedToday(dataset_id, isMasterDataset) const processingDefaultThreshold = await SystemConfig.getThresholds("processing") @@ -123,16 +114,16 @@ export const getProcessingHealth = async (dataset: any): Promise<{ components: a avgHealth = HealthStatus.UnHealthy } } - const failure = await getValidationFailure(dataset_id) + const failure = await queryMetrics(dataset_id, prometheusQueries.validationFailure) failure.health = getProcessingComponentHealth(failure, count, processingDefaultThreshold?.validationFailuresCount) - const dedupFailure = await getDedupFailure(dataset_id) + const dedupFailure = await queryMetrics(dataset_id, prometheusQueries.dedupFailure) dedupFailure.health = getProcessingComponentHealth(dedupFailure, count, processingDefaultThreshold?.dedupFailuresCount) - const denormFailure = await getDenormFailure(dataset_id) + const denormFailure = await queryMetrics(dataset_id, prometheusQueries.denormFailure) denormFailure.health = getProcessingComponentHealth(denormFailure, count, processingDefaultThreshold?.denormFailureCount) - const transformFailure = await getTransformFailure(dataset_id) + const transformFailure = await queryMetrics(dataset_id, prometheusQueries.transformationFailure) denormFailure.health = getProcessingComponentHealth(transformFailure, count, processingDefaultThreshold?.transformFailureCount) const components = [ @@ -197,7 +188,7 @@ const getProcessingComponentHealth = (info: any, count: any, threshold: any) => export const getQueryHealth = async (datasources: any, dataset: any): Promise<{ components: any, status: string }> => { const components: any = []; - const isMasterDataset = _.get(dataset, "type") == DatasetType.MasterDataset; + const isMasterDataset = _.get(dataset, "type") == DatasetType.master; let status = HealthStatus.Healthy; if (!isMasterDataset) { if (!_.isEmpty(datasources)) { @@ -223,7 +214,7 @@ export const getQueryHealth = async (datasources: any, dataset: any): Promise<{ } - const queriesCount = await getQuriesStatus(dataset?.dataset_id) + const queriesCount = await queryMetrics(dataset?.dataset_id, prometheusQueries.queriesCount) const defaultThresholds = await SystemConfig.getThresholds("query") components.push({ @@ -232,7 +223,7 @@ export const getQueryHealth = async (datasources: any, dataset: any): Promise<{ "status": queriesCount.health }) - const avgQueryReponseTimeInSec = await getAvgQueryReponseTimeInSec(dataset?.dataset_id) + const avgQueryReponseTimeInSec = await queryMetrics(dataset?.dataset_id, prometheusQueries.avgQueryResponseTimeInSec) if (avgQueryReponseTimeInSec.count > defaultThresholds?.avgQueryReponseTimeInSec) { avgQueryReponseTimeInSec.health = HealthStatus.UnHealthy } @@ -242,7 +233,7 @@ export const getQueryHealth = async (datasources: any, dataset: any): Promise<{ "status": avgQueryReponseTimeInSec.health }) - const queriesFailed = await getQueriesFailedCount(dataset?.dataset_id) + const queriesFailed = await queryMetrics(dataset?.dataset_id, prometheusQueries.queriesFailedCount) if (queriesCount.count == 0 && queriesFailed.count > 0) { queriesFailed.health = HealthStatus.UnHealthy } else { @@ -300,17 +291,38 @@ const getDruidDataourceStatus = async (datasourceId: string) => { const getPostgresStatus = async (): Promise => { try { - const postgresStatus = await postgresHealth() - logger.debug(postgresStatus) + await postgresHealth() } catch (error) { - logger.error("errr: ", error) + logger.error(error) return HealthStatus.UnHealthy } return HealthStatus.Healthy } -const getRedisStatus = async () => { - return isRedisDenormHealthy && isRedisDedupHealthy ? HealthStatus.Healthy : HealthStatus.UnHealthy +const connectToRedis = async (url: string) => { + return new Promise((resolve, reject) => { + createClient({ + url + }) + .on("error", (err: any) => { + reject(err) + }) + .on("connect", () => { + resolve("connected") + }) + .connect(); + }) +} + +const getRedisHealthStatus = async () => { + try { + await Promise.all([connectToRedis(`redis://${config.redis_config.denorm_redis_host}:${config.redis_config.denorm_redis_port}`), + connectToRedis(`redis://${config.redis_config.dedup_redis_host}:${config.redis_config.dedup_redis_port}`)]); + return HealthStatus.Healthy; + } catch (error) { + logger.error(error) + } + return HealthStatus.UnHealthy; } const getKafkaHealthStatus = async () => { @@ -345,7 +357,7 @@ const getDruidHealthStatus = async () => { const { data = false } = await druidHttpService.get("/status/health") return data ? HealthStatus.Healthy : HealthStatus.UnHealthy } catch (error) { - logger.error("druid health check", error) + logger.error(error) return HealthStatus.UnHealthy } } @@ -470,86 +482,4 @@ const getAvgProcessingSpeedInSec = async (datasetId: string, isMasterDataset: bo logger.error(error) return { count: 0, health: HealthStatus.UnHealthy } } -} - -const getValidationFailure = async (datasetId: string) => { - const query = `sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_${getDatasetIdForMetrics(datasetId)}_validator_failed_count[1d]))` - try { - const { data } = await queryMetrics({ query }) - return { count: _.toInteger(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } - } catch (error) { - logger.error(error) - return { count: 0, health: HealthStatus.UnHealthy } - } -} - -const getDedupFailure = async (datasetId: string) => { - const query = `sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_${getDatasetIdForMetrics(datasetId)}_dedup_failed_count[1d]))`; - try { - const { data } = await queryMetrics({ query }) - return { count: _.toInteger(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } - } catch (error) { - logger.error(error) - return { count: 0, health: HealthStatus.UnHealthy } - } -} - -const getDenormFailure = async (datasetId: string) => { - const query = `sum(sum_over_time(flink_taskmanager_job_task_operator_DenormalizerJob_${getDatasetIdForMetrics(datasetId)}_denorm_failed[1d]))`; - try { - const { data } = await queryMetrics({ query }) - return { count: _.toInteger(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } - } catch (error) { - logger.error(error) - return { count: 0, health: HealthStatus.UnHealthy } - } -} - -const getTransformFailure = async (datasetId: string) => { - const query = `sum(sum_over_time(flink_taskmanager_job_task_operator_TransformerJob_${getDatasetIdForMetrics(datasetId)}_transform_failed_count[1d]))`; - try { - const { data } = await queryMetrics({ query }) - return { count: _.toInteger(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } - } catch (error) { - logger.error(error) - return { count: 0, health: HealthStatus.UnHealthy } - } -} - -const getQuriesStatus = async (datasetId: string) => { - const query = `sum(sum_over_time(node_total_api_calls{entity="data-out", dataset_id="${getDatasetIdForMetrics(datasetId)}"}[1d]))`; - try { - const { data } = await queryMetrics({ query }) - logger.debug(data) - return { count: _.toInteger(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } - } catch (error) { - logger.error(error) - return { count: 0, health: HealthStatus.UnHealthy } - } -} - -const getAvgQueryReponseTimeInSec = async (datasetId: string) => { - const query = `avg(avg_over_time(node_query_response_time{entity='data-out', dataset_id="${getDatasetIdForMetrics(datasetId)}"}[1d]))/1000`; - try { - const { data } = await queryMetrics({ query }) - logger.debug(data) - return { count: +(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } - } catch (error) { - logger.error(error) - return { count: 0, health: HealthStatus.UnHealthy } - } -} - -const getQueriesFailedCount = async (datasetId: string) => { - const query = `sum(sum_over_time(node_failed_api_calls{entity='data-out', dataset_id="${getDatasetIdForMetrics(datasetId)}"}[1d]))`; - try { - const { data } = await queryMetrics({ query }) - logger.debug(data) - return { count: _.toInteger(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } - } catch (error) { - logger.error(error) - return { count: 0, health: HealthStatus.UnHealthy } - } -} - -init().catch(err => logger.error(err)) \ No newline at end of file +} \ No newline at end of file From 104d238413b0b5d99f0c3b34d86e02c1c6944135 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Wed, 24 Jul 2024 16:07:04 +0530 Subject: [PATCH 052/235] #OBS-I101: Update the publish API for the v2 APIs --- .../DatasetCreateValidationSchema.json | 9 +- .../ReadyToPublishSchema.json | 10 +- .../DatasetUpdateValidationSchema.json | 11 +- api-service/src/v2/services/DatasetService.ts | 8 +- .../src/command/dataset_command.py | 13 +- command-service/src/command/db_command.py | 363 ++++++++++-------- command-service/src/model/db_models.py | 38 +- command-service/src/service/db_service.py | 14 +- 8 files changed, 262 insertions(+), 204 deletions(-) diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json index 938fca6d..055efdc3 100644 --- a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -278,6 +278,10 @@ "items": { "type": "object", "properties": { + "id": { + "type": "string", + "minLength": 1 + }, "connector_id": { "type": "string", "minLength": 1 @@ -287,10 +291,13 @@ }, "operations_config": { "type": "object" + }, + "version": { + "type": "string" } }, "additionalProperties": false, - "required": ["connector_id", "connector_config"] + "required": ["id", "connector_id", "connector_config", "version"] } }, "tags": { diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index 10f0ca91..7a789977 100644 --- a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -367,6 +367,10 @@ "items": { "type": "object", "properties": { + "id": { + "type": "string", + "minLength": 1 + }, "connector_id": { "type": "string", "minLength": 1 @@ -376,10 +380,14 @@ }, "operations_config": { "type": "object" + }, + "version": { + "type": "string", + "minLength": 1 } }, "additionalProperties": false, - "required": ["connector_id", "connector_config"] + "required": ["id", "connector_id", "connector_config", "version"] } } }, diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 929ac0bb..25bd2c98 100644 --- a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -327,6 +327,10 @@ "value": { "type": "object", "properties": { + "id": { + "type": "string", + "minLength": 1 + }, "connector_id": { "type": "string", "minLength": 1 @@ -336,9 +340,12 @@ }, "operations_config": { "type": "object" + }, + "version": { + "type": "string" } }, - "required": ["connector_id"], + "required": ["id"], "additionalProperties": false }, "action": { @@ -358,7 +365,7 @@ "then": { "properties": { "value": { - "required": ["connector_config"] + "required": ["connector_id", "connector_config"] } } } diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/v2/services/DatasetService.ts index 43bbcdf0..52ca8cf9 100644 --- a/api-service/src/v2/services/DatasetService.ts +++ b/api-service/src/v2/services/DatasetService.ts @@ -113,9 +113,10 @@ class DatasetService { category: this.getTransformationCategory(_.get(config, ["metadata.section"])) } }) - const connectors = await this.getDraftConnectors(datasetId, ["connector_type", "connector_config"]); + const connectors = await this.getDraftConnectors(datasetId, ["id", "connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { return { + id: _.get(config, ["id"]), connector_id: _.get(config, ["connector_type"]), connector_config: _.get(config, ["connector_config"]), version: "v1" @@ -158,9 +159,10 @@ class DatasetService { keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} } - const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["connector_type", "connector_config"]); + const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { return { + id: _.get(config, "id"), connector_id: _.get(config, "connector_type"), connector_config: _.get(config, "connector_config"), version: "v1" @@ -178,7 +180,7 @@ class DatasetService { }) draftDataset["api_version"] = "v2" } else { - const connectors = await this.getConnectors(draftDataset.dataset_id, ["connector_id", "connector_config", "operations_config"]); + const connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); draftDataset["connectors_config"] = connectors const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "datatype", "category"]); draftDataset["transformations_config"] = transformations diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index edd147e8..165efd38 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -2,7 +2,6 @@ import time from dacite import from_dict - from command.icommand import ICommand from config import Config from model.data_models import CommandPayload, DatasetStatusType @@ -12,7 +11,6 @@ from service.http_service import HttpService from service.telemetry_service import TelemetryService - class DatasetCommand(ICommand): def __init__( self, @@ -38,6 +36,17 @@ def _get_draft_dataset_record(self, dataset_id): if dataset_record is not None: return dataset_record return None + + def _get_draft_dataset(self, dataset_id): + query = f""" + SELECT * FROM datasets_draft + WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}') AND version = (SELECT MAX(version) + FROM datasets_draft WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}')) + """ + dataset_record = self.db_service.execute_select_one(query) + if dataset_record is not None: + return dataset_record + return None def _check_for_live_record(self, dataset_id): query = f""" diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index b428c3e5..55ed86dc 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -1,21 +1,13 @@ import json import time from datetime import datetime as dt - from dacite import from_dict - from command.dataset_command import DatasetCommand from command.icommand import ICommand from model.data_models import Action, ActionResponse, CommandPayload, DatasetStatusType -from model.db_models import ( - DatasetsDraft, - DatasetSourceConfigDraft, - DatasetTransformationsDraft, - DatasourcesDraft, -) +from model.db_models import DatasetsDraft, DatasetConnectorConfigDraft, DatasourcesDraft, DatasetTransformationsDraft from service.db_service import DatabaseService - class DBCommand(ICommand): def __init__(self, db_service: DatabaseService, dataset_command: DatasetCommand): @@ -23,6 +15,7 @@ def __init__(self, db_service: DatabaseService, dataset_command: DatasetCommand) self.dataset_command = dataset_command def execute(self, command_payload: CommandPayload, action: Action): + if action == Action.MAKE_DATASET_LIVE.name: print( f"Invoking MAKE_DATASET_LIVE command for dataset_id {command_payload.dataset_id}..." @@ -30,8 +23,13 @@ def execute(self, command_payload: CommandPayload, action: Action): result = self._change_dataset_to_active(command_payload) print(f"Result from MAKE_DATASET_ACTIVE {result}...") return result + else: + return ActionResponse( + status="ERROR", status_code=404, error_message="INVALID_ACTION" + ) def _change_dataset_to_active(self, command_payload: CommandPayload): + dataset_id = command_payload.dataset_id live_dataset, data_version = self.dataset_command._check_for_live_record( dataset_id @@ -40,224 +38,255 @@ def _change_dataset_to_active(self, command_payload: CommandPayload): self.dataset_command.audit_live_dataset( command_payload, int(time.time() * 1000) ) + + draft_dataset_record = self.dataset_command._get_draft_dataset(dataset_id) + draft_dataset_id = self._insert_dataset_record( - dataset_id, data_version, live_dataset + dataset_id, data_version, live_dataset, draft_dataset_record ) if draft_dataset_id: self._insert_datasource_record(dataset_id, draft_dataset_id) - self._insert_dataset_source_config(dataset_id, draft_dataset_id) - self._insert_dataset_transformations(dataset_id, draft_dataset_id) + self._insert_connector_instances(dataset_id, draft_dataset_record) + self._insert_dataset_transformations(dataset_id, draft_dataset_record) + self._delete_draft_dataset(dataset_id, draft_dataset_id) return ActionResponse(status="OK", status_code=200) else: return ActionResponse( status="ERROR", status_code=404, error_message="DATASET_ID_NOT_FOUND" ) - def _insert_dataset_record(self, dataset_id, data_version, live_dataset): - query = f""" - SELECT * FROM datasets_draft - WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}') AND version = (SELECT MAX(version) - FROM datasets_draft WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}')) - """ - draft_dataset_record = self.db_service.execute_select_one(query) - draft_dataset_id = "" - if draft_dataset_record is not None: - draft_dataset = from_dict( - data_class=DatasetsDraft, data=draft_dataset_record + def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_dataset_record): + + if draft_dataset_record is None: + return None + + draft_dataset = from_dict(data_class = DatasetsDraft, data = draft_dataset_record) + draft_dataset_id = draft_dataset.id + current_timestamp = dt.now() + insert_query = f""" + INSERT INTO datasets(id, dataset_id, "type", name, extraction_config, validation_config, dedup_config, + denorm_config, data_schema, router_config, dataset_config, status, tags, data_version, api_version, + version, sample_data, entry_topic, created_by, updated_by, created_date, updated_date, published_date) + VALUES ( + '{dataset_id}', + '{dataset_id}', + '{draft_dataset.type}', + '{draft_dataset.name}', + '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', + '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', + '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', + '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', + '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', + '{json.dumps(draft_dataset.router_config).replace("'", "''")}', + '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', + '{DatasetStatusType.Live.name}', + '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', + 1, + '{draft_dataset.api_version}', + {draft_dataset.version}, + '{json.dumps(draft_dataset.sample_data).replace("'", "''")}', + '{draft_dataset.entry_topic}', + '{draft_dataset.created_by}', + '{draft_dataset.updated_by}', + '{current_timestamp}', + '{current_timestamp}', + '{current_timestamp}' ) - draft_dataset_id = draft_dataset.id + ON CONFLICT (id) DO UPDATE + SET name = '{draft_dataset.name}', + extraction_config = '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', + validation_config = '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', + dedup_config = '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', + denorm_config = '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', + data_schema = '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', + router_config = '{json.dumps(draft_dataset.router_config).replace("'", "''")}', + dataset_config = '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', + tags = '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', + data_version = {data_version if live_dataset is not None else 1}, + api_version = '{draft_dataset.api_version}', + version = {draft_dataset.version}, + sample_data = '{json.dumps(draft_dataset.sample_data).replace("'", "''")}', + entry_topic = '{draft_dataset.entry_topic}', + updated_by = '{draft_dataset.updated_by}', + updated_date = '{current_timestamp}', + published_date = '{current_timestamp}', + status = '{DatasetStatusType.Live.name}'; + """ + self.db_service.execute_upsert(insert_query) + print(f"Dataset {dataset_id} record inserted successfully...") + return draft_dataset_id + + def _insert_datasource_record(self, dataset_id, draft_dataset_id): + + result = {} + draft_datasource_record = self.db_service.execute_select_all( + f"SELECT * FROM datasources_draft WHERE dataset_id = '{draft_dataset_id}'" + ) + if draft_datasource_record is None: + return result + for record in draft_datasource_record: + draft_datasource = from_dict(data_class=DatasourcesDraft, data=record) current_timestamp = dt.now() insert_query = f""" - INSERT INTO datasets(id, dataset_id, "type", name, extraction_config, validation_config, dedup_config, - denorm_config, data_schema, router_config, dataset_config, status, tags, data_version, created_by, updated_by, created_date, - updated_date, published_date) + INSERT INTO datasources(id, datasource, dataset_id, datasource_ref, ingestion_spec, type, retention_period, + archival_policy, purge_policy, backup_config, status, created_by, updated_by, created_date, + updated_date, published_date, metadata) VALUES ( + '{draft_datasource.id}', + '{draft_datasource.datasource}', '{dataset_id}', - '{dataset_id}', - '{draft_dataset.type}', - '{draft_dataset.name}', - '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', - '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', - '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', - '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', - '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', - '{json.dumps(draft_dataset.router_config).replace("'", "''")}', - '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', + '{draft_datasource.datasource_ref}', + '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', + '{draft_datasource.type}', + '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', + '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', + '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', + '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', '{DatasetStatusType.Live.name}', - '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', - {draft_dataset.version}, - '{draft_dataset.created_by}', - '{draft_dataset.updated_by}', + '{draft_datasource.created_by}', + '{draft_datasource.updated_by}', '{current_timestamp}', '{current_timestamp}', - '{current_timestamp}' + '{current_timestamp}', + '{json.dumps(draft_datasource.metadata).replace("'", "''")}' ) ON CONFLICT (id) DO UPDATE - SET name = '{draft_dataset.name}', - extraction_config = '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', - validation_config = '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', - dedup_config = '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', - denorm_config = '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', - data_schema = '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', - router_config = '{json.dumps(draft_dataset.router_config).replace("'", "''")}', - dataset_config = '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', - tags = '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', - data_version = {data_version if live_dataset is not None else draft_dataset.version}, - updated_by = '{draft_dataset.updated_by}', + SET datasource_ref = '{draft_datasource.datasource}', + ingestion_spec = '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', + type = '{draft_datasource.type}', + retention_period = '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', + archival_policy = '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', + purge_policy = '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', + backup_config = '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', + updated_by = '{draft_datasource.updated_by}', updated_date = '{current_timestamp}', published_date = '{current_timestamp}', + metadata = '{json.dumps(draft_datasource.metadata).replace("'", "''")}', status = '{DatasetStatusType.Live.name}'; - """ - self.db_service.execute_upsert(insert_query) - print(f"Dataset {dataset_id} record inserted successfully...") - self.db_service.execute_upsert( - f"UPDATE datasets_draft SET status = '{DatasetStatusType.Live.name}', published_date = now() WHERE id = '{draft_dataset_id}'" + """ + result = self.db_service.execute_upsert(insert_query) + print( + f"Datasource {draft_datasource.id} record inserted successfully..." ) + return result - return draft_dataset_id - - def _insert_datasource_record(self, dataset_id, draft_dataset_id): - result = {} - draft_datasource_record = self.db_service.execute_select_all( - f"SELECT * FROM datasources_draft WHERE dataset_id = '{draft_dataset_id}'" - ) - if draft_datasource_record is not None: - for record in draft_datasource_record: - draft_datasource = from_dict(data_class=DatasourcesDraft, data=record) - current_timestamp = dt.now() + def _insert_connector_instances(self, dataset_id, draft_dataset_record): + emptyJson, result = {} + draft_connectors_config_record = draft_dataset_record.connectors_config + if draft_connectors_config_record is None: + return result + + for record in draft_connectors_config_record: + connector_config = from_dict( + data_class = DatasetConnectorConfigDraft, data = record + ) + current_timestamp = dt.now() + if connector_config.version == 'v2': insert_query = f""" - INSERT INTO datasources(id, datasource, dataset_id, datasource_ref, ingestion_spec, type, retention_period, - archival_policy, purge_policy, backup_config, status, created_by, updated_by, created_date, - updated_date, published_date, metadata) + INSERT INTO connector_instances(id, dataset_id, connector_id, connector_config, operations_config, + data_format, status, connector_state, connector_stats, created_by, updated_by, created_date, + updated_date, published_date) VALUES ( - '{dataset_id + '_' + draft_datasource.datasource}', - '{draft_datasource.datasource}', + '{connector_config.id}', '{dataset_id}', - '{draft_datasource.datasource}', - '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', - '{draft_datasource.type}', - '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', - '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', - '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', - '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', + '{connector_config.connector_id}', + '{json.dumps(connector_config.connector_config).replace("'", "''")}', + '{json.dumps(connector_config.operations_config).replace("'", "''")}', + '{connector_config.data_format}', '{DatasetStatusType.Live.name}', - '{draft_datasource.created_by}', - '{draft_datasource.updated_by}', + '{json.dumps(emptyJson)}', + '{json.dumps(emptyJson)}', + '{draft_dataset_record.created_by}', + '{draft_dataset_record.updated_by}', '{current_timestamp}', '{current_timestamp}', - '{current_timestamp}', - '{json.dumps(draft_datasource.metadata).replace("'", "''")}' + '{current_timestamp}' ) ON CONFLICT (id) DO UPDATE - SET datasource_ref = '{draft_datasource.datasource}', - ingestion_spec = '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', - type = '{draft_datasource.type}', - retention_period = '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', - archival_policy = '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', - purge_policy = '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', - backup_config = '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', - updated_by = '{draft_datasource.updated_by}', + SET connector_config = '{json.dumps(connector_config.connector_config).replace("'", "''")}', + operations_config = '{json.dumps(connector_config.operations_config).replace("'", "''")}', + data_format = '{connector_config.data_format}', + updated_by = '{draft_dataset_record.updated_by}', updated_date = '{current_timestamp}', published_date = '{current_timestamp}', - metadata = '{json.dumps(draft_datasource.metadata).replace("'", "''")}', status = '{DatasetStatusType.Live.name}'; """ result = self.db_service.execute_upsert(insert_query) print( - f"Datasource {dataset_id + '_' + draft_datasource.datasource} record inserted successfully..." + f"Connector[v2] Instance record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." ) - update_result = self.db_service.execute_upsert( - f"UPDATE datasources_draft SET status = '{DatasetStatusType.Live.name}', published_date = now() WHERE dataset_id = '{draft_dataset_id}'" - ) - return result - - def _insert_dataset_source_config(self, dataset_id, draft_dataset_id): - draft_dataset_source_config_record = self.db_service.execute_select_all( - f"SELECT * FROM dataset_source_config_draft WHERE dataset_id = '{draft_dataset_id}'" - ) - result = {} - if draft_dataset_source_config_record is not None: - for record in draft_dataset_source_config_record: - draft_dataset_source_config = from_dict( - data_class=DatasetSourceConfigDraft, data=record - ) - current_timestamp = dt.now() + else: insert_query = f""" - INSERT INTO dataset_source_config(id, dataset_id, connector_type, connector_config, connector_stats, + INSERT INTO dataset_source_config(id, dataset_id, connector_type, connector_config, status, created_by, updated_by, created_date, updated_date, published_date) VALUES ( - '{draft_dataset_source_config.id}', + '{connector_config.id}', '{dataset_id}', - '{draft_dataset_source_config.connector_type}', - '{json.dumps(draft_dataset_source_config.connector_config).replace("'", "''")}', - '{json.dumps(draft_dataset_source_config.connector_stats).replace("'", "''")}', + '{connector_config.connector_id}', + '{json.dumps(connector_config.connector_config).replace("'", "''")}', '{DatasetStatusType.Live.name}', - '{draft_dataset_source_config.created_by}', - '{draft_dataset_source_config.updated_by}', + '{draft_dataset_record.created_by}', + '{draft_dataset_record.updated_by}', '{current_timestamp}', '{current_timestamp}', '{current_timestamp}' ) ON CONFLICT (id) DO UPDATE - SET connector_type = '{draft_dataset_source_config.connector_type}', - connector_config = '{json.dumps(draft_dataset_source_config.connector_config).replace("'", "''")}', - connector_stats = '{json.dumps(draft_dataset_source_config.connector_stats).replace("'", "''")}', - updated_by = '{draft_dataset_source_config.updated_by}', + SET connector_config = '{json.dumps(connector_config.connector_config).replace("'", "''")}', + updated_by = '{draft_dataset_record.updated_by}', updated_date = '{current_timestamp}', published_date = '{current_timestamp}', status = '{DatasetStatusType.Live.name}'; """ result = self.db_service.execute_upsert(insert_query) print( - f"Dataset Source Config {dataset_id + '_' + draft_dataset_source_config.connector_type} record inserted successfully..." + f"Connector[v1] record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." ) - update_result = self.db_service.execute_upsert( - f"UPDATE dataset_source_config_draft SET status = '{DatasetStatusType.Live.name}', published_date = now() WHERE dataset_id = '{draft_dataset_id}'" - ) + return result - def _insert_dataset_transformations(self, dataset_id, draft_dataset_id): - draft_dataset_transformations_record = self.db_service.execute_select_all( - f"SELECT * FROM dataset_transformations_draft WHERE dataset_id = '{draft_dataset_id}'" - ) + def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): + + draft_dataset_transformations_record = draft_dataset_record.transformations_config result = {} current_timestamp = dt.now() - if draft_dataset_transformations_record is not None: - for record in draft_dataset_transformations_record: - draft_dataset_transformations = from_dict( - data_class=DatasetTransformationsDraft, data=record - ) - insert_query = f""" - INSERT INTO dataset_transformations(id, dataset_id, field_key, transformation_function, - status, mode, created_by, updated_by, created_date, updated_date, published_date, metadata) - VALUES ( - '{dataset_id + '_' + draft_dataset_transformations.field_key}', - '{dataset_id}', - '{draft_dataset_transformations.field_key}', - '{json.dumps(draft_dataset_transformations.transformation_function).replace("'", "''")}', - '{DatasetStatusType.Live.name}', - '{draft_dataset_transformations.mode}', - '{draft_dataset_transformations.created_by}', - '{draft_dataset_transformations.updated_by}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}', - '{json.dumps(draft_dataset_transformations.metadata).replace("'", "''")}' - ) - ON CONFLICT (id) DO UPDATE - SET transformation_function = '{json.dumps(draft_dataset_transformations.transformation_function).replace("'", "''")}', - updated_by = '{draft_dataset_transformations.updated_by}', - updated_date = '{current_timestamp}', - published_date = '{current_timestamp}', - metadata = '{json.dumps(draft_dataset_transformations.metadata).replace("'", "''")}', - mode = '{draft_dataset_transformations.mode}', - status = '{DatasetStatusType.Live.name}'; - """ - result = self.db_service.execute_upsert(insert_query) - print( - f"Dataset Transformation {dataset_id + '_' + draft_dataset_transformations.field_key} record inserted successfully..." - ) - update_result = self.db_service.execute_upsert( - f"UPDATE dataset_transformations_draft SET status = '{DatasetStatusType.Live.name}', published_date = now() WHERE dataset_id = '{draft_dataset_id}'" + # Delete existing transformations + self.db_service.execute_delete(f"""DELETE from dataset_transformations where dataset_id = '{dataset_id}'""") + print(f"Dataset Transformation for {dataset_id} are deleted successfully...") + + if draft_dataset_transformations_record is None: + return result + + for record in draft_dataset_transformations_record: + transformation = from_dict( + data_class=DatasetTransformationsDraft, data=record ) + insert_query = f""" + INSERT INTO dataset_transformations(id, dataset_id, field_key, transformation_function, + status, mode, created_by, updated_by, created_date, updated_date, published_date, metadata) + VALUES ( + '{dataset_id + '_' + transformation.field_key}', + '{dataset_id}', + '{transformation.field_key}', + '{json.dumps(transformation.transformation_function).replace("'", "''")}', + '{DatasetStatusType.Live.name}', + '{transformation.mode}', + '{transformation.created_by}', + '{transformation.updated_by}', + '{current_timestamp}', + '{current_timestamp}', + '{current_timestamp}', + '{json.dumps(transformation.metadata).replace("'", "''")}' + ) + """ + result = self.db_service.execute_upsert(insert_query) + print(f"Dataset Transformation {dataset_id + '_' + transformation.field_key} record inserted successfully...") return result + + def _delete_draft_dataset(self, dataset_id, draft_dataset_id): + + self.db_service.execute_delete(f"""DELETE from datasources_draft where dataset_id = '{draft_dataset_id}'""") + print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(f"""DELETE from datasets_draft where id = '{draft_dataset_id}'""") + print(f"Draft Dataset for {dataset_id} is deleted successfully...") \ No newline at end of file diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index a8c12c8d..4ecce478 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -1,9 +1,7 @@ from dataclasses import dataclass from datetime import datetime - from dataclasses_json import dataclass_json - @dataclass class DatasetsLive: id: str @@ -17,6 +15,7 @@ class DatasetsLive: data_schema: dict router_config: dict dataset_config: dict + version: int status: str created_by: str created_date: datetime @@ -41,12 +40,17 @@ class DatasetsDraft: router_config: dict dataset_config: dict status: str + api_version: str + transformations_config: dict | None = None + connectors_config: dict | None = None + denorm_config: dict | None = None + sample_data: dict + entry_topic: str created_by: str created_date: datetime tags: list[str] | None = None updated_by: str | None = None updated_date: datetime | None = None - denorm_config: dict | None = None published_date: datetime | None = None @@ -72,34 +76,20 @@ class DatasourcesDraft: @dataclass -class DatasetSourceConfigDraft: +class DatasetConnectorConfigDraft: id: str - dataset_id: str - connector_type: str - status: str - created_by: str - created_date: datetime - connector_config: dict | None = None - connector_stats: dict | None = None - updated_by: str | None = None - updated_date: datetime | None = None - published_date: datetime | None = None + connector_id: str + connector_config: dict + operations_config: dict | None = None + data_format: str | None = 'json' + version: str @dataclass class DatasetTransformationsDraft: - id: str - dataset_id: str field_key: str transformation_function: dict - status: str mode: str - created_by: str - created_date: datetime - updated_by: str | None = None - updated_date: datetime | None = None - published_date: datetime | None = None - metadata: dict | None = None @dataclass_json @@ -133,4 +123,4 @@ class ConnectorInstance: operations_config: dict connector_runtime: str connector_source: str - technology: str + technology: str \ No newline at end of file diff --git a/command-service/src/service/db_service.py b/command-service/src/service/db_service.py index 97aeb313..864b4612 100644 --- a/command-service/src/service/db_service.py +++ b/command-service/src/service/db_service.py @@ -1,13 +1,12 @@ -from typing import Callable - import psycopg2 import psycopg2.extras -from tenacity import retry, stop_after_attempt, wait_exponential +from typing import Callable +from tenacity import retry, stop_after_attempt, wait_exponential from config import Config - def reconnect(func: Callable): + def wrapper(db_connection, *args, **kwargs): tdecorator = retry(wait=wait_exponential(), stop=stop_after_attempt(3)) decorated = tdecorator(func) @@ -65,3 +64,10 @@ def execute_upsert(self, sql): db_connection.close() # print(f"{record_count} inserted/updated successfully") return record_count + +# @reconnect + def execute_delete(self, sql): + db_connection = self.connect() + cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + cursor.execute(sql) + db_connection.close() From abc4ad2d6c04cf0b6927a3fc9d0887dd6b22f2a4 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Wed, 24 Jul 2024 17:41:30 +0530 Subject: [PATCH 053/235] #OBS-I101: fix db models --- command-service/src/model/db_models.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index 4ecce478..1d915d21 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -41,13 +41,13 @@ class DatasetsDraft: dataset_config: dict status: str api_version: str - transformations_config: dict | None = None - connectors_config: dict | None = None - denorm_config: dict | None = None - sample_data: dict entry_topic: str created_by: str created_date: datetime + sample_data: dict | None = None + transformations_config: dict | None = None + connectors_config: dict | None = None + denorm_config: dict | None = None tags: list[str] | None = None updated_by: str | None = None updated_date: datetime | None = None @@ -80,9 +80,9 @@ class DatasetConnectorConfigDraft: id: str connector_id: str connector_config: dict + version: str operations_config: dict | None = None data_format: str | None = 'json' - version: str @dataclass @@ -116,6 +116,7 @@ class ConnectorRegsitryv2: updated_by: str | None = None livedate: datetime | None = None + @dataclass class ConnectorInstance: id: str From f088bb038089c9d91be55c4962045de78de4cc5b Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Wed, 24 Jul 2024 18:48:52 +0530 Subject: [PATCH 054/235] #OBS-I115: Remove the v1 unused API code and restructure the folders --- api-service/src/app.ts | 49 +- api-service/src/{v2 => }/configs/Config.ts | 0 .../src/{v2 => }/configs/ConnectionsConfig.ts | 0 .../{v2 => }/configs/DatasetConfigDefault.ts | 0 .../src/{v2 => }/configs/IngestionConfig.ts | 0 .../src/{v1 => }/configs/QueryRules.ts | 0 .../connections/commandServiceConnection.ts | 0 .../connections/databaseConnection.ts | 0 .../{v2 => }/connections/druidConnection.ts | 0 .../{v2 => }/connections/kafkaConnection.ts | 0 .../CreateTemplateController.ts | 0 .../CreateTemplateValidationSchema.json | 0 .../QueryTemplateValidator.ts | 0 .../DataExhaust/DataExhaustController.ts | 0 .../DataIngestion/DataIngestionController.ts | 0 .../DataIngestion/validationSchema.json | 0 .../controllers/DataOut/DataOutController.ts | 0 .../DataOut/DataOutValidationSchema.json | 0 .../controllers/DataOut/QueryRules.ts | 0 .../controllers/DataOut/QueryValidator.ts | 0 .../DatasetCreate/DatasetCreate.ts | 0 .../DatasetCreateValidationSchema.json | 0 .../controllers/DatasetList/DatasetList.ts | 0 .../DatasetListValidationSchema.json | 0 .../controllers/DatasetRead/DatasetRead.ts | 0 .../DatasetStatusTransition.ts | 0 .../ReadyToPublishSchema.json | 0 .../RequestValidationSchema.json | 0 .../DatasetUpdate/DatasetUpdate.ts | 0 .../DatasetUpdateValidationSchema.json | 0 .../DeleteTemplateController.ts | 0 .../EventValidation/EventValidation.ts | 0 .../RequestValidationSchema.json | 0 .../GenerateDataSchema/GenerateDataSchema.ts | 0 .../RequestValidationSchema.json | 0 .../GenerateSignedURL/GenerateSignedURL.ts | 0 .../GenerateSignedURLValidationSchema.json | 0 .../ListTemplateValidationSchema.json | 0 .../ListTemplatesController.ts | 0 .../QueryTemplate/QueryTemplateController.ts | 0 .../QueryTemplate/QueryTemplateHelpers.ts | 0 .../QueryTemplateValidationSchema.json | 0 .../QueryWrapper/SqlQueryWrapper.ts | 0 .../ReadTemplateController.ts | 0 .../UpdateTemplateController.ts | 0 .../UpdateTemplateValidationSchema.json | 0 .../exceptions/SchemaGenerationException.ts | 0 .../{v2 => }/helpers/ErrorResponseHandler.ts | 0 .../src/{v2 => }/helpers/ResponseHandler.ts | 5 + api-service/src/{v2 => }/logger/index.ts | 0 .../{v2 => }/metrics/prometheus/entities.ts | 0 .../{v2 => }/metrics/prometheus/helpers.ts | 0 .../src/{v2 => }/metrics/prometheus/index.ts | 0 .../{v2 => }/metrics/prometheus/metrics.ts | 0 .../src/{v2 => }/middlewares/errors.ts | 0 .../middlewares/setDataToRequestObject.ts | 0 .../src/{v2 => }/models/ConnectorInstances.ts | 0 .../src/{v2 => }/models/ConnectorRegistry.ts | 0 api-service/src/{v2 => }/models/Dataset.ts | 0 .../src/{v2 => }/models/DatasetDraft.ts | 0 .../{v2 => }/models/DatasetSourceConfig.ts | 0 .../models/DatasetSourceConfigDraft.ts | 0 api-service/src/{v2 => }/models/Datasource.ts | 0 .../src/{v2 => }/models/DatasourceDraft.ts | 0 .../src/{v2 => }/models/QueryTemplate.ts | 0 .../src/{v2 => }/models/Transformation.ts | 0 .../{v2 => }/models/TransformationDraft.ts | 0 api-service/src/routes/DruidProxyRouter.ts | 23 + .../MetricRouter.ts} | 4 +- api-service/src/{v2 => }/routes/Router.ts | 4 +- .../src/{v2 => }/services/CipherService.ts | 0 .../CloudServices/AWSStorageService.ts | 0 .../CloudServices/AzureStorageService.ts | 0 .../CloudServices/GCPStorageService.ts | 0 .../{v2 => }/services/CloudServices/index.ts | 0 .../{v2 => }/services/CloudServices/types.ts | 0 .../src/{v2 => }/services/DatasetService.ts | 0 .../services/DatasetSourceConfigService.ts | 0 .../{v2 => }/services/DatasourceService.ts | 0 api-service/src/services/HealthService.ts | 20 + .../{v2 => }/services/QueryTemplateService.ts | 0 .../SchemaGenerateService/ConfigSuggester.ts | 0 .../SchemaGenerateService/Constants.json | 0 .../DataSchemaService.ts | 0 .../SchemaGenerateService/SchemaAnalyser.ts | 0 .../SchemaArrayValidator.ts | 0 .../SchemaCardinalityAnalyser.ts | 0 .../SchemaGeneratorUtils.ts | 0 .../SchemaGenerateService/SchemaHandler.ts | 0 .../SchemaGenerateService/SchemaMapping.json | 0 .../SchemaGenerateService/SchemaMerger.ts | 0 .../SuggestionTemplate.ts | 0 .../SchemaGenerateService/Template.ts | 0 .../src/{v2 => }/services/TableGenerator.ts | 0 .../{v2 => }/services/ValidationService.ts | 0 .../src/{v1 => }/services/WrapperService.ts | 5 +- .../src/{v2 => }/services/telemetry.ts | 0 .../data => telemetry}/telemetryActions.ts | 0 .../DataIngestTest/DataIngestionTest.spec.ts | 0 .../DataIngestTest/Fixtures.ts | 0 .../DataOutTest/DataQueryTest.spec.ts | 0 .../DatasetManagement/DataOutTest/Fixtures.ts | 0 .../DatasetCreate/DatasetCreate.spec.ts | 0 .../DatasetCreate/Fixtures.ts | 0 .../DatasetList/DatasetList.spec.ts | 0 .../DatasetManagement/DatasetList/Fixtures.ts | 0 .../DatasetRead/DatasetRead.spec.ts | 0 .../DatasetManagement/DatasetRead/Fixtures.ts | 0 .../DatasetDelete.spec.ts | 0 .../DatasetLive.spec.ts | 0 .../DatasetReadyToPublish.spec.ts | 0 .../DatasetRetire.spec.ts | 0 .../DatasetStatusTransition.spec.ts | 0 .../DatasetStatusTransition/Fixtures.ts | 0 .../DatasetUpdate/DatasetDedup.spec.ts | 0 .../DatasetUpdate/DatasetDenorm.spec.ts | 0 .../DatasetUpdate/DatasetExtraction.spec.ts | 0 .../DatasetUpdate/DatasetTags.spec.ts | 0 .../DatasetTransformation.spec.ts | 0 .../DatasetUpdate/DatasetUpdate.spec.ts | 0 .../DatasetUpdate/DatasetValidation.spec.ts | 0 .../DatasetUpdate/Fixtures.ts | 0 .../GenerateSignedURL/Fixtures.ts | 0 .../GenerateSignedURL.spec.ts | 0 .../CreateTemplate/CreateTemplate.spec.ts | 0 .../QueryTemplates/CreateTemplate/Fixtures.ts | 0 .../DeleteTemplate/DeleteTemplate.spec.ts | 0 .../QueryTemplates/ListTemplates/Fixtures.ts | 0 .../ListTemplates/ListTemplates.spec.ts | 0 .../ReadTemplate/ReadTemplate.spec.ts | 0 .../TemplateQuerying/Fixtures.ts | 0 .../TemplateQuerying/TemplateQuerying.spec.ts | 0 .../QueryTemplates/UpdateTemplate/Fixtures.ts | 0 .../UpdateTemplate/UpdateTemplate.spec.ts | 0 .../tests/QueryWrapper/SqlWrapper/Fixtures.ts | 0 .../SqlWrapper/SqlWrapper.spec.ts | 0 .../src/{v2 => }/types/ConfigModels.ts | 0 .../{v1/models => types}/ConnectionModels.ts | 0 .../src/{v2 => }/types/DatasetModels.ts | 0 .../src/{v1/models => types}/ExhaustModels.ts | 0 .../src/{v2 => }/types/IngestionModels.ts | 0 api-service/src/{v2 => }/types/MetricModel.ts | 2 +- api-service/src/{v2 => }/types/ObsrvError.ts | 0 .../src/{v1/models => types}/QueryModels.ts | 0 .../src/{v2 => }/types/ResponseModel.ts | 0 .../src/{v2 => }/types/SampleURLModel.ts | 0 api-service/src/{v2 => }/types/SchemaModel.ts | 0 .../{v1/models => types}/ValidationModels.ts | 0 api-service/src/{v2 => }/utils/common.ts | 0 api-service/src/v1/configs/Config.ts | 109 --- api-service/src/v1/configs/Extensions.ts | 11 - api-service/src/v1/configs/RoutesConfig.ts | 181 ----- api-service/src/v1/connectors/DbConnector.ts | 179 ----- .../src/v1/connectors/HttpConnector.ts | 28 - .../src/v1/connectors/KafkaConnector.ts | 25 - api-service/src/v1/generators/SchemaMerger.ts | 7 - api-service/src/v1/helpers/DatasetConfigs.ts | 19 - .../src/v1/helpers/DatasetSourceConfigs.ts | 52 -- api-service/src/v1/helpers/Datasets.ts | 97 --- api-service/src/v1/helpers/Datasources.ts | 65 -- api-service/src/v1/helpers/DbUtil.ts | 60 -- .../src/v1/helpers/ErrorResponseHandler.ts | 32 - api-service/src/v1/helpers/LakehouseUtil.ts | 51 -- api-service/src/v1/helpers/ResponseHandler.ts | 53 -- .../src/v1/helpers/ValidationService.ts | 12 - .../AWSStorageService.js | 475 ----------- .../AzureStorageService.js | 405 ---------- .../BaseStorageService.js | 109 --- .../GCPStorageService.js | 278 ------- .../src/v1/lib/client-cloud-services/index.js | 34 - .../client-cloud-services/storageLogger.js | 56 -- .../src/v1/lib/dispatcher/dispatcher.js | 54 -- .../src/v1/lib/dispatcher/kafka-dispatcher.js | 60 -- api-service/src/v1/lib/envVariables.js | 15 - .../src/v1/lib/services/TelemetryService.js | 124 --- api-service/src/v1/managers/Extensions.ts | 28 - api-service/src/v1/models/ConfigModels.ts | 44 -- api-service/src/v1/models/DatasetModels.ts | 57 -- api-service/src/v1/models/IngestionModels.ts | 35 - api-service/src/v1/resources/Constants.json | 111 --- .../v1/resources/schemas/DataExhaustReq.json | 21 - .../resources/schemas/DataIngestionReq.json | 11 - .../resources/schemas/DatasetConfigDefault.ts | 87 --- .../resources/schemas/DatasetCreateReq.json | 99 --- .../v1/resources/schemas/DatasetListReq.json | 9 - .../schemas/DatasetSourceConfigSaveReq.json | 33 - .../schemas/DatasetSourceConfigUpdateReq.json | 33 - .../resources/schemas/DatasetUpdateReq.json | 99 --- .../schemas/DatasourceConfigDefault.json | 22 - .../resources/schemas/DatasourceSaveReq.json | 128 --- .../schemas/DatasourceUpdateReq.json | 130 --- .../v1/resources/schemas/QueryRequest.json | 49 -- .../resources/schemas/SchemaValidatorReq.json | 30 - .../resources/schemas/SubmitIngestionReq.json | 12 - api-service/src/v1/routes/Router.ts | 86 -- .../src/v1/services/ClientCloudService.ts | 74 -- .../src/v1/services/DataSourceService.ts | 81 -- api-service/src/v1/services/DatasetService.ts | 58 -- .../v1/services/DatasetSourceConfigService.ts | 53 -- .../services/EventsValidationAgainstSchema.ts | 32 - api-service/src/v1/services/HealthService.ts | 46 -- .../src/v1/services/IngestorService.ts | 105 --- api-service/src/v1/services/QueryService.ts | 49 -- .../src/v1/services/ValidationService.ts | 37 - api-service/src/v1/services/telemetry.ts | 187 ----- .../src/v1/test/ClientCloudService.spec.ts | 128 --- api-service/src/v1/test/Config.ts | 31 - .../DatasetSourceConfigTestService.spec.ts | 300 ------- .../src/v1/test/DatasetTestService.spec.ts | 366 --------- .../src/v1/test/DatasourceTestService.spec.ts | 493 ------------ api-service/src/v1/test/Fixtures.ts | 154 ---- .../src/v1/test/IngestorTestService.spec.ts | 276 ------- .../src/v1/test/QueryTestService.spec.ts | 738 ------------------ api-service/src/v1/test/TestConnector.spec.ts | 193 ----- api-service/src/v1/utils/common.ts | 38 - .../src/v1/validators/QueryValidator.ts | 228 ------ .../src/v1/validators/RequestsValidator.ts | 87 --- api-service/src/v2/configs/QueryRules.ts | 56 -- api-service/src/v2/services/HealthService.ts | 45 -- .../src/v2/telemetry/telemetryActions.ts | 21 - api-service/src/v2/types/ConnectionModels.ts | 13 - api-service/src/v2/types/ExhaustModels.ts | 4 - api-service/src/v2/types/QueryModels.ts | 66 -- api-service/src/v2/types/ValidationModels.ts | 6 - .../{v2 => }/validators/RequestsValidator.ts | 0 225 files changed, 72 insertions(+), 7720 deletions(-) rename api-service/src/{v2 => }/configs/Config.ts (100%) rename api-service/src/{v2 => }/configs/ConnectionsConfig.ts (100%) rename api-service/src/{v2 => }/configs/DatasetConfigDefault.ts (100%) rename api-service/src/{v2 => }/configs/IngestionConfig.ts (100%) rename api-service/src/{v1 => }/configs/QueryRules.ts (100%) rename api-service/src/{v2 => }/connections/commandServiceConnection.ts (100%) rename api-service/src/{v2 => }/connections/databaseConnection.ts (100%) rename api-service/src/{v2 => }/connections/druidConnection.ts (100%) rename api-service/src/{v2 => }/connections/kafkaConnection.ts (100%) rename api-service/src/{v2 => }/controllers/CreateQueryTemplate/CreateTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/CreateQueryTemplate/QueryTemplateValidator.ts (100%) rename api-service/src/{v2 => }/controllers/DataExhaust/DataExhaustController.ts (100%) rename api-service/src/{v2 => }/controllers/DataIngestion/DataIngestionController.ts (100%) rename api-service/src/{v2 => }/controllers/DataIngestion/validationSchema.json (100%) rename api-service/src/{v2 => }/controllers/DataOut/DataOutController.ts (100%) rename api-service/src/{v2 => }/controllers/DataOut/DataOutValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/DataOut/QueryRules.ts (100%) rename api-service/src/{v2 => }/controllers/DataOut/QueryValidator.ts (100%) rename api-service/src/{v2 => }/controllers/DatasetCreate/DatasetCreate.ts (100%) rename api-service/src/{v2 => }/controllers/DatasetCreate/DatasetCreateValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/DatasetList/DatasetList.ts (100%) rename api-service/src/{v2 => }/controllers/DatasetList/DatasetListValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/DatasetRead/DatasetRead.ts (100%) rename api-service/src/{v2 => }/controllers/DatasetStatusTransition/DatasetStatusTransition.ts (100%) rename api-service/src/{v2 => }/controllers/DatasetStatusTransition/ReadyToPublishSchema.json (100%) rename api-service/src/{v2 => }/controllers/DatasetStatusTransition/RequestValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/DatasetUpdate/DatasetUpdate.ts (100%) rename api-service/src/{v2 => }/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/DeleteQueryTemplate/DeleteTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/EventValidation/EventValidation.ts (100%) rename api-service/src/{v2 => }/controllers/EventValidation/RequestValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/GenerateDataSchema/GenerateDataSchema.ts (100%) rename api-service/src/{v2 => }/controllers/GenerateDataSchema/RequestValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/GenerateSignedURL/GenerateSignedURL.ts (100%) rename api-service/src/{v2 => }/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/ListQueryTemplates/ListTemplateValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/ListQueryTemplates/ListTemplatesController.ts (100%) rename api-service/src/{v2 => }/controllers/QueryTemplate/QueryTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/QueryTemplate/QueryTemplateHelpers.ts (100%) rename api-service/src/{v2 => }/controllers/QueryTemplate/QueryTemplateValidationSchema.json (100%) rename api-service/src/{v2 => }/controllers/QueryWrapper/SqlQueryWrapper.ts (100%) rename api-service/src/{v2 => }/controllers/ReadQueryTemplate/ReadTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/UpdateQueryTemplate/UpdateTemplateController.ts (100%) rename api-service/src/{v2 => }/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json (100%) rename api-service/src/{v2 => }/exceptions/SchemaGenerationException.ts (100%) rename api-service/src/{v2 => }/helpers/ErrorResponseHandler.ts (100%) rename api-service/src/{v2 => }/helpers/ResponseHandler.ts (89%) rename api-service/src/{v2 => }/logger/index.ts (100%) rename api-service/src/{v2 => }/metrics/prometheus/entities.ts (100%) rename api-service/src/{v2 => }/metrics/prometheus/helpers.ts (100%) rename api-service/src/{v2 => }/metrics/prometheus/index.ts (100%) rename api-service/src/{v2 => }/metrics/prometheus/metrics.ts (100%) rename api-service/src/{v2 => }/middlewares/errors.ts (100%) rename api-service/src/{v2 => }/middlewares/setDataToRequestObject.ts (100%) rename api-service/src/{v2 => }/models/ConnectorInstances.ts (100%) rename api-service/src/{v2 => }/models/ConnectorRegistry.ts (100%) rename api-service/src/{v2 => }/models/Dataset.ts (100%) rename api-service/src/{v2 => }/models/DatasetDraft.ts (100%) rename api-service/src/{v2 => }/models/DatasetSourceConfig.ts (100%) rename api-service/src/{v2 => }/models/DatasetSourceConfigDraft.ts (100%) rename api-service/src/{v2 => }/models/Datasource.ts (100%) rename api-service/src/{v2 => }/models/DatasourceDraft.ts (100%) rename api-service/src/{v2 => }/models/QueryTemplate.ts (100%) rename api-service/src/{v2 => }/models/Transformation.ts (100%) rename api-service/src/{v2 => }/models/TransformationDraft.ts (100%) create mode 100644 api-service/src/routes/DruidProxyRouter.ts rename api-service/src/{v2/routes/metricRouter.ts => routes/MetricRouter.ts} (56%) rename api-service/src/{v2 => }/routes/Router.ts (99%) rename api-service/src/{v2 => }/services/CipherService.ts (100%) rename api-service/src/{v2 => }/services/CloudServices/AWSStorageService.ts (100%) rename api-service/src/{v2 => }/services/CloudServices/AzureStorageService.ts (100%) rename api-service/src/{v2 => }/services/CloudServices/GCPStorageService.ts (100%) rename api-service/src/{v2 => }/services/CloudServices/index.ts (100%) rename api-service/src/{v2 => }/services/CloudServices/types.ts (100%) rename api-service/src/{v2 => }/services/DatasetService.ts (100%) rename api-service/src/{v2 => }/services/DatasetSourceConfigService.ts (100%) rename api-service/src/{v2 => }/services/DatasourceService.ts (100%) create mode 100644 api-service/src/services/HealthService.ts rename api-service/src/{v2 => }/services/QueryTemplateService.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/ConfigSuggester.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/Constants.json (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/DataSchemaService.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/SchemaAnalyser.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/SchemaArrayValidator.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/SchemaGeneratorUtils.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/SchemaHandler.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/SchemaMapping.json (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/SchemaMerger.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/SuggestionTemplate.ts (100%) rename api-service/src/{v2 => }/services/SchemaGenerateService/Template.ts (100%) rename api-service/src/{v2 => }/services/TableGenerator.ts (100%) rename api-service/src/{v2 => }/services/ValidationService.ts (100%) rename api-service/src/{v1 => }/services/WrapperService.ts (98%) rename api-service/src/{v2 => }/services/telemetry.ts (100%) rename api-service/src/{v1/data => telemetry}/telemetryActions.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DataIngestTest/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DataOutTest/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetCreate/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetList/DatasetList.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetList/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetRead/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/DatasetUpdate/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/CreateTemplate/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/ListTemplates/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/TemplateQuerying/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/UpdateTemplate/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts (100%) rename api-service/src/{v2 => }/tests/QueryWrapper/SqlWrapper/Fixtures.ts (100%) rename api-service/src/{v2 => }/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts (100%) rename api-service/src/{v2 => }/types/ConfigModels.ts (100%) rename api-service/src/{v1/models => types}/ConnectionModels.ts (100%) rename api-service/src/{v2 => }/types/DatasetModels.ts (100%) rename api-service/src/{v1/models => types}/ExhaustModels.ts (100%) rename api-service/src/{v2 => }/types/IngestionModels.ts (100%) rename api-service/src/{v2 => }/types/MetricModel.ts (90%) rename api-service/src/{v2 => }/types/ObsrvError.ts (100%) rename api-service/src/{v1/models => types}/QueryModels.ts (100%) rename api-service/src/{v2 => }/types/ResponseModel.ts (100%) rename api-service/src/{v2 => }/types/SampleURLModel.ts (100%) rename api-service/src/{v2 => }/types/SchemaModel.ts (100%) rename api-service/src/{v1/models => types}/ValidationModels.ts (100%) rename api-service/src/{v2 => }/utils/common.ts (100%) delete mode 100644 api-service/src/v1/configs/Config.ts delete mode 100644 api-service/src/v1/configs/Extensions.ts delete mode 100644 api-service/src/v1/configs/RoutesConfig.ts delete mode 100644 api-service/src/v1/connectors/DbConnector.ts delete mode 100644 api-service/src/v1/connectors/HttpConnector.ts delete mode 100644 api-service/src/v1/connectors/KafkaConnector.ts delete mode 100644 api-service/src/v1/generators/SchemaMerger.ts delete mode 100644 api-service/src/v1/helpers/DatasetConfigs.ts delete mode 100644 api-service/src/v1/helpers/DatasetSourceConfigs.ts delete mode 100644 api-service/src/v1/helpers/Datasets.ts delete mode 100644 api-service/src/v1/helpers/Datasources.ts delete mode 100644 api-service/src/v1/helpers/DbUtil.ts delete mode 100644 api-service/src/v1/helpers/ErrorResponseHandler.ts delete mode 100644 api-service/src/v1/helpers/LakehouseUtil.ts delete mode 100644 api-service/src/v1/helpers/ResponseHandler.ts delete mode 100644 api-service/src/v1/helpers/ValidationService.ts delete mode 100644 api-service/src/v1/lib/client-cloud-services/AWSStorageService.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/AzureStorageService.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/BaseStorageService.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/GCPStorageService.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/index.js delete mode 100644 api-service/src/v1/lib/client-cloud-services/storageLogger.js delete mode 100644 api-service/src/v1/lib/dispatcher/dispatcher.js delete mode 100644 api-service/src/v1/lib/dispatcher/kafka-dispatcher.js delete mode 100644 api-service/src/v1/lib/envVariables.js delete mode 100644 api-service/src/v1/lib/services/TelemetryService.js delete mode 100644 api-service/src/v1/managers/Extensions.ts delete mode 100644 api-service/src/v1/models/ConfigModels.ts delete mode 100644 api-service/src/v1/models/DatasetModels.ts delete mode 100644 api-service/src/v1/models/IngestionModels.ts delete mode 100644 api-service/src/v1/resources/Constants.json delete mode 100644 api-service/src/v1/resources/schemas/DataExhaustReq.json delete mode 100644 api-service/src/v1/resources/schemas/DataIngestionReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetConfigDefault.ts delete mode 100644 api-service/src/v1/resources/schemas/DatasetCreateReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetListReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetSourceConfigSaveReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetSourceConfigUpdateReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasetUpdateReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasourceConfigDefault.json delete mode 100644 api-service/src/v1/resources/schemas/DatasourceSaveReq.json delete mode 100644 api-service/src/v1/resources/schemas/DatasourceUpdateReq.json delete mode 100644 api-service/src/v1/resources/schemas/QueryRequest.json delete mode 100644 api-service/src/v1/resources/schemas/SchemaValidatorReq.json delete mode 100644 api-service/src/v1/resources/schemas/SubmitIngestionReq.json delete mode 100644 api-service/src/v1/routes/Router.ts delete mode 100644 api-service/src/v1/services/ClientCloudService.ts delete mode 100644 api-service/src/v1/services/DataSourceService.ts delete mode 100644 api-service/src/v1/services/DatasetService.ts delete mode 100644 api-service/src/v1/services/DatasetSourceConfigService.ts delete mode 100644 api-service/src/v1/services/EventsValidationAgainstSchema.ts delete mode 100644 api-service/src/v1/services/HealthService.ts delete mode 100644 api-service/src/v1/services/IngestorService.ts delete mode 100644 api-service/src/v1/services/QueryService.ts delete mode 100644 api-service/src/v1/services/ValidationService.ts delete mode 100644 api-service/src/v1/services/telemetry.ts delete mode 100644 api-service/src/v1/test/ClientCloudService.spec.ts delete mode 100644 api-service/src/v1/test/Config.ts delete mode 100644 api-service/src/v1/test/DatasetSourceConfigTestService.spec.ts delete mode 100644 api-service/src/v1/test/DatasetTestService.spec.ts delete mode 100644 api-service/src/v1/test/DatasourceTestService.spec.ts delete mode 100644 api-service/src/v1/test/Fixtures.ts delete mode 100644 api-service/src/v1/test/IngestorTestService.spec.ts delete mode 100644 api-service/src/v1/test/QueryTestService.spec.ts delete mode 100644 api-service/src/v1/test/TestConnector.spec.ts delete mode 100644 api-service/src/v1/utils/common.ts delete mode 100644 api-service/src/v1/validators/QueryValidator.ts delete mode 100644 api-service/src/v1/validators/RequestsValidator.ts delete mode 100644 api-service/src/v2/configs/QueryRules.ts delete mode 100644 api-service/src/v2/services/HealthService.ts delete mode 100644 api-service/src/v2/telemetry/telemetryActions.ts delete mode 100644 api-service/src/v2/types/ConnectionModels.ts delete mode 100644 api-service/src/v2/types/ExhaustModels.ts delete mode 100644 api-service/src/v2/types/QueryModels.ts delete mode 100644 api-service/src/v2/types/ValidationModels.ts rename api-service/src/{v2 => }/validators/RequestsValidator.ts (100%) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 66cbaa4d..ab864a98 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -1,45 +1,28 @@ import express, { Application } from "express"; -import { config } from "./v1/configs/Config"; -import { ResponseHandler } from "./v1/helpers/ResponseHandler"; -import { loadExtensions } from "./v1/managers/Extensions"; -import { router } from "./v1/routes/Router"; -import {router as v2Router} from "./v2/routes/Router" -import {router as metricsRouter} from "./v2/routes/metricRouter" +import {router as v2Router} from "./routes/Router" +import { metricRouter } from "./routes/MetricRouter" +import { druidProxyRouter } from "./routes/DruidProxyRouter" + import bodyParser from "body-parser"; -import { interceptAuditEvents } from "./v1/services/telemetry"; -import { queryService } from "./v1/routes/Router"; -import { routesConfig } from "./v1/configs/RoutesConfig"; -import { QueryValidator } from "./v1/validators/QueryValidator"; -import { errorHandler, obsrvErrorHandler } from "./v2/middlewares/errors"; -const app: Application = express(); -const queryValidator = new QueryValidator(); +import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; +import { ResponseHandler } from "./helpers/ResponseHandler"; +import { config } from "./configs/Config"; -const services = { - queryService, - validationService: queryValidator, - nativeQueryId: routesConfig.query.native_query.api_id, - sqlQueryId: routesConfig.query.sql_query.api_id, -} +const app: Application = express(); app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.use(errorHandler) -app.set("queryServices", services); -loadExtensions(app) - .finally(() => { - app.use(interceptAuditEvents()) - app.use("/v2/", v2Router); - app.use("/", router); - app.use("/", metricsRouter); - app.use("*", ResponseHandler.routeNotFound); - app.use(obsrvErrorHandler); +app.use("/v2/", v2Router); +app.use("/", druidProxyRouter); +app.use("/", metricRouter); +app.use("*", ResponseHandler.routeNotFound); +app.use(obsrvErrorHandler); - app.listen(config.api_port, () => { - console.log(`listening on port ${config.api_port}`); - }); +app.listen(config.api_port, () => { + console.log(`listening on port ${config.api_port}`); }); - -export default app; +export default app; \ No newline at end of file diff --git a/api-service/src/v2/configs/Config.ts b/api-service/src/configs/Config.ts similarity index 100% rename from api-service/src/v2/configs/Config.ts rename to api-service/src/configs/Config.ts diff --git a/api-service/src/v2/configs/ConnectionsConfig.ts b/api-service/src/configs/ConnectionsConfig.ts similarity index 100% rename from api-service/src/v2/configs/ConnectionsConfig.ts rename to api-service/src/configs/ConnectionsConfig.ts diff --git a/api-service/src/v2/configs/DatasetConfigDefault.ts b/api-service/src/configs/DatasetConfigDefault.ts similarity index 100% rename from api-service/src/v2/configs/DatasetConfigDefault.ts rename to api-service/src/configs/DatasetConfigDefault.ts diff --git a/api-service/src/v2/configs/IngestionConfig.ts b/api-service/src/configs/IngestionConfig.ts similarity index 100% rename from api-service/src/v2/configs/IngestionConfig.ts rename to api-service/src/configs/IngestionConfig.ts diff --git a/api-service/src/v1/configs/QueryRules.ts b/api-service/src/configs/QueryRules.ts similarity index 100% rename from api-service/src/v1/configs/QueryRules.ts rename to api-service/src/configs/QueryRules.ts diff --git a/api-service/src/v2/connections/commandServiceConnection.ts b/api-service/src/connections/commandServiceConnection.ts similarity index 100% rename from api-service/src/v2/connections/commandServiceConnection.ts rename to api-service/src/connections/commandServiceConnection.ts diff --git a/api-service/src/v2/connections/databaseConnection.ts b/api-service/src/connections/databaseConnection.ts similarity index 100% rename from api-service/src/v2/connections/databaseConnection.ts rename to api-service/src/connections/databaseConnection.ts diff --git a/api-service/src/v2/connections/druidConnection.ts b/api-service/src/connections/druidConnection.ts similarity index 100% rename from api-service/src/v2/connections/druidConnection.ts rename to api-service/src/connections/druidConnection.ts diff --git a/api-service/src/v2/connections/kafkaConnection.ts b/api-service/src/connections/kafkaConnection.ts similarity index 100% rename from api-service/src/v2/connections/kafkaConnection.ts rename to api-service/src/connections/kafkaConnection.ts diff --git a/api-service/src/v2/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/CreateQueryTemplate/CreateTemplateController.ts rename to api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts diff --git a/api-service/src/v2/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json rename to api-service/src/controllers/CreateQueryTemplate/CreateTemplateValidationSchema.json diff --git a/api-service/src/v2/controllers/CreateQueryTemplate/QueryTemplateValidator.ts b/api-service/src/controllers/CreateQueryTemplate/QueryTemplateValidator.ts similarity index 100% rename from api-service/src/v2/controllers/CreateQueryTemplate/QueryTemplateValidator.ts rename to api-service/src/controllers/CreateQueryTemplate/QueryTemplateValidator.ts diff --git a/api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts b/api-service/src/controllers/DataExhaust/DataExhaustController.ts similarity index 100% rename from api-service/src/v2/controllers/DataExhaust/DataExhaustController.ts rename to api-service/src/controllers/DataExhaust/DataExhaustController.ts diff --git a/api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts similarity index 100% rename from api-service/src/v2/controllers/DataIngestion/DataIngestionController.ts rename to api-service/src/controllers/DataIngestion/DataIngestionController.ts diff --git a/api-service/src/v2/controllers/DataIngestion/validationSchema.json b/api-service/src/controllers/DataIngestion/validationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DataIngestion/validationSchema.json rename to api-service/src/controllers/DataIngestion/validationSchema.json diff --git a/api-service/src/v2/controllers/DataOut/DataOutController.ts b/api-service/src/controllers/DataOut/DataOutController.ts similarity index 100% rename from api-service/src/v2/controllers/DataOut/DataOutController.ts rename to api-service/src/controllers/DataOut/DataOutController.ts diff --git a/api-service/src/v2/controllers/DataOut/DataOutValidationSchema.json b/api-service/src/controllers/DataOut/DataOutValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DataOut/DataOutValidationSchema.json rename to api-service/src/controllers/DataOut/DataOutValidationSchema.json diff --git a/api-service/src/v2/controllers/DataOut/QueryRules.ts b/api-service/src/controllers/DataOut/QueryRules.ts similarity index 100% rename from api-service/src/v2/controllers/DataOut/QueryRules.ts rename to api-service/src/controllers/DataOut/QueryRules.ts diff --git a/api-service/src/v2/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts similarity index 100% rename from api-service/src/v2/controllers/DataOut/QueryValidator.ts rename to api-service/src/controllers/DataOut/QueryValidator.ts diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts similarity index 100% rename from api-service/src/v2/controllers/DatasetCreate/DatasetCreate.ts rename to api-service/src/controllers/DatasetCreate/DatasetCreate.ts diff --git a/api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DatasetCreate/DatasetCreateValidationSchema.json rename to api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json diff --git a/api-service/src/v2/controllers/DatasetList/DatasetList.ts b/api-service/src/controllers/DatasetList/DatasetList.ts similarity index 100% rename from api-service/src/v2/controllers/DatasetList/DatasetList.ts rename to api-service/src/controllers/DatasetList/DatasetList.ts diff --git a/api-service/src/v2/controllers/DatasetList/DatasetListValidationSchema.json b/api-service/src/controllers/DatasetList/DatasetListValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DatasetList/DatasetListValidationSchema.json rename to api-service/src/controllers/DatasetList/DatasetListValidationSchema.json diff --git a/api-service/src/v2/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts similarity index 100% rename from api-service/src/v2/controllers/DatasetRead/DatasetRead.ts rename to api-service/src/controllers/DatasetRead/DatasetRead.ts diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts similarity index 100% rename from api-service/src/v2/controllers/DatasetStatusTransition/DatasetStatusTransition.ts rename to api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json similarity index 100% rename from api-service/src/v2/controllers/DatasetStatusTransition/ReadyToPublishSchema.json rename to api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json diff --git a/api-service/src/v2/controllers/DatasetStatusTransition/RequestValidationSchema.json b/api-service/src/controllers/DatasetStatusTransition/RequestValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DatasetStatusTransition/RequestValidationSchema.json rename to api-service/src/controllers/DatasetStatusTransition/RequestValidationSchema.json diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts similarity index 100% rename from api-service/src/v2/controllers/DatasetUpdate/DatasetUpdate.ts rename to api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts diff --git a/api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json rename to api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json diff --git a/api-service/src/v2/controllers/DeleteQueryTemplate/DeleteTemplateController.ts b/api-service/src/controllers/DeleteQueryTemplate/DeleteTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/DeleteQueryTemplate/DeleteTemplateController.ts rename to api-service/src/controllers/DeleteQueryTemplate/DeleteTemplateController.ts diff --git a/api-service/src/v2/controllers/EventValidation/EventValidation.ts b/api-service/src/controllers/EventValidation/EventValidation.ts similarity index 100% rename from api-service/src/v2/controllers/EventValidation/EventValidation.ts rename to api-service/src/controllers/EventValidation/EventValidation.ts diff --git a/api-service/src/v2/controllers/EventValidation/RequestValidationSchema.json b/api-service/src/controllers/EventValidation/RequestValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/EventValidation/RequestValidationSchema.json rename to api-service/src/controllers/EventValidation/RequestValidationSchema.json diff --git a/api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts b/api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts similarity index 100% rename from api-service/src/v2/controllers/GenerateDataSchema/GenerateDataSchema.ts rename to api-service/src/controllers/GenerateDataSchema/GenerateDataSchema.ts diff --git a/api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json b/api-service/src/controllers/GenerateDataSchema/RequestValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/GenerateDataSchema/RequestValidationSchema.json rename to api-service/src/controllers/GenerateDataSchema/RequestValidationSchema.json diff --git a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts similarity index 100% rename from api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURL.ts rename to api-service/src/controllers/GenerateSignedURL/GenerateSignedURL.ts diff --git a/api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json b/api-service/src/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json rename to api-service/src/controllers/GenerateSignedURL/GenerateSignedURLValidationSchema.json diff --git a/api-service/src/v2/controllers/ListQueryTemplates/ListTemplateValidationSchema.json b/api-service/src/controllers/ListQueryTemplates/ListTemplateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/ListQueryTemplates/ListTemplateValidationSchema.json rename to api-service/src/controllers/ListQueryTemplates/ListTemplateValidationSchema.json diff --git a/api-service/src/v2/controllers/ListQueryTemplates/ListTemplatesController.ts b/api-service/src/controllers/ListQueryTemplates/ListTemplatesController.ts similarity index 100% rename from api-service/src/v2/controllers/ListQueryTemplates/ListTemplatesController.ts rename to api-service/src/controllers/ListQueryTemplates/ListTemplatesController.ts diff --git a/api-service/src/v2/controllers/QueryTemplate/QueryTemplateController.ts b/api-service/src/controllers/QueryTemplate/QueryTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/QueryTemplate/QueryTemplateController.ts rename to api-service/src/controllers/QueryTemplate/QueryTemplateController.ts diff --git a/api-service/src/v2/controllers/QueryTemplate/QueryTemplateHelpers.ts b/api-service/src/controllers/QueryTemplate/QueryTemplateHelpers.ts similarity index 100% rename from api-service/src/v2/controllers/QueryTemplate/QueryTemplateHelpers.ts rename to api-service/src/controllers/QueryTemplate/QueryTemplateHelpers.ts diff --git a/api-service/src/v2/controllers/QueryTemplate/QueryTemplateValidationSchema.json b/api-service/src/controllers/QueryTemplate/QueryTemplateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/QueryTemplate/QueryTemplateValidationSchema.json rename to api-service/src/controllers/QueryTemplate/QueryTemplateValidationSchema.json diff --git a/api-service/src/v2/controllers/QueryWrapper/SqlQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts similarity index 100% rename from api-service/src/v2/controllers/QueryWrapper/SqlQueryWrapper.ts rename to api-service/src/controllers/QueryWrapper/SqlQueryWrapper.ts diff --git a/api-service/src/v2/controllers/ReadQueryTemplate/ReadTemplateController.ts b/api-service/src/controllers/ReadQueryTemplate/ReadTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/ReadQueryTemplate/ReadTemplateController.ts rename to api-service/src/controllers/ReadQueryTemplate/ReadTemplateController.ts diff --git a/api-service/src/v2/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts similarity index 100% rename from api-service/src/v2/controllers/UpdateQueryTemplate/UpdateTemplateController.ts rename to api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts diff --git a/api-service/src/v2/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json rename to api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateValidationSchema.json diff --git a/api-service/src/v2/exceptions/SchemaGenerationException.ts b/api-service/src/exceptions/SchemaGenerationException.ts similarity index 100% rename from api-service/src/v2/exceptions/SchemaGenerationException.ts rename to api-service/src/exceptions/SchemaGenerationException.ts diff --git a/api-service/src/v2/helpers/ErrorResponseHandler.ts b/api-service/src/helpers/ErrorResponseHandler.ts similarity index 100% rename from api-service/src/v2/helpers/ErrorResponseHandler.ts rename to api-service/src/helpers/ErrorResponseHandler.ts diff --git a/api-service/src/v2/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts similarity index 89% rename from api-service/src/v2/helpers/ResponseHandler.ts rename to api-service/src/helpers/ResponseHandler.ts index faeb3847..7f166db0 100644 --- a/api-service/src/v2/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -55,6 +55,11 @@ const ResponseHandler = { entity && onSuccess(req, res) res.status(result.status).send(result.data); }, + + goneResponse: (req: Request, res: Response) => { + const { id, entity } = req as any; + res.status(httpStatus.GONE).json({ id: id, ver: "v1", ts: Date.now(), params: { status: "FAILED", errmsg: "v1 APIs have been replace by /v2 APIs. Please refer to this link for more information" }, responseCode: httpStatus["410_NAME"] }) + } } export { ResponseHandler }; diff --git a/api-service/src/v2/logger/index.ts b/api-service/src/logger/index.ts similarity index 100% rename from api-service/src/v2/logger/index.ts rename to api-service/src/logger/index.ts diff --git a/api-service/src/v2/metrics/prometheus/entities.ts b/api-service/src/metrics/prometheus/entities.ts similarity index 100% rename from api-service/src/v2/metrics/prometheus/entities.ts rename to api-service/src/metrics/prometheus/entities.ts diff --git a/api-service/src/v2/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts similarity index 100% rename from api-service/src/v2/metrics/prometheus/helpers.ts rename to api-service/src/metrics/prometheus/helpers.ts diff --git a/api-service/src/v2/metrics/prometheus/index.ts b/api-service/src/metrics/prometheus/index.ts similarity index 100% rename from api-service/src/v2/metrics/prometheus/index.ts rename to api-service/src/metrics/prometheus/index.ts diff --git a/api-service/src/v2/metrics/prometheus/metrics.ts b/api-service/src/metrics/prometheus/metrics.ts similarity index 100% rename from api-service/src/v2/metrics/prometheus/metrics.ts rename to api-service/src/metrics/prometheus/metrics.ts diff --git a/api-service/src/v2/middlewares/errors.ts b/api-service/src/middlewares/errors.ts similarity index 100% rename from api-service/src/v2/middlewares/errors.ts rename to api-service/src/middlewares/errors.ts diff --git a/api-service/src/v2/middlewares/setDataToRequestObject.ts b/api-service/src/middlewares/setDataToRequestObject.ts similarity index 100% rename from api-service/src/v2/middlewares/setDataToRequestObject.ts rename to api-service/src/middlewares/setDataToRequestObject.ts diff --git a/api-service/src/v2/models/ConnectorInstances.ts b/api-service/src/models/ConnectorInstances.ts similarity index 100% rename from api-service/src/v2/models/ConnectorInstances.ts rename to api-service/src/models/ConnectorInstances.ts diff --git a/api-service/src/v2/models/ConnectorRegistry.ts b/api-service/src/models/ConnectorRegistry.ts similarity index 100% rename from api-service/src/v2/models/ConnectorRegistry.ts rename to api-service/src/models/ConnectorRegistry.ts diff --git a/api-service/src/v2/models/Dataset.ts b/api-service/src/models/Dataset.ts similarity index 100% rename from api-service/src/v2/models/Dataset.ts rename to api-service/src/models/Dataset.ts diff --git a/api-service/src/v2/models/DatasetDraft.ts b/api-service/src/models/DatasetDraft.ts similarity index 100% rename from api-service/src/v2/models/DatasetDraft.ts rename to api-service/src/models/DatasetDraft.ts diff --git a/api-service/src/v2/models/DatasetSourceConfig.ts b/api-service/src/models/DatasetSourceConfig.ts similarity index 100% rename from api-service/src/v2/models/DatasetSourceConfig.ts rename to api-service/src/models/DatasetSourceConfig.ts diff --git a/api-service/src/v2/models/DatasetSourceConfigDraft.ts b/api-service/src/models/DatasetSourceConfigDraft.ts similarity index 100% rename from api-service/src/v2/models/DatasetSourceConfigDraft.ts rename to api-service/src/models/DatasetSourceConfigDraft.ts diff --git a/api-service/src/v2/models/Datasource.ts b/api-service/src/models/Datasource.ts similarity index 100% rename from api-service/src/v2/models/Datasource.ts rename to api-service/src/models/Datasource.ts diff --git a/api-service/src/v2/models/DatasourceDraft.ts b/api-service/src/models/DatasourceDraft.ts similarity index 100% rename from api-service/src/v2/models/DatasourceDraft.ts rename to api-service/src/models/DatasourceDraft.ts diff --git a/api-service/src/v2/models/QueryTemplate.ts b/api-service/src/models/QueryTemplate.ts similarity index 100% rename from api-service/src/v2/models/QueryTemplate.ts rename to api-service/src/models/QueryTemplate.ts diff --git a/api-service/src/v2/models/Transformation.ts b/api-service/src/models/Transformation.ts similarity index 100% rename from api-service/src/v2/models/Transformation.ts rename to api-service/src/models/Transformation.ts diff --git a/api-service/src/v2/models/TransformationDraft.ts b/api-service/src/models/TransformationDraft.ts similarity index 100% rename from api-service/src/v2/models/TransformationDraft.ts rename to api-service/src/models/TransformationDraft.ts diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts new file mode 100644 index 00000000..ca1aec91 --- /dev/null +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -0,0 +1,23 @@ +import express from "express"; +import { Entity } from "../types/MetricModel"; +import { wrapperService } from "../services/WrapperService"; +import { onRequest } from "../metrics/prometheus/helpers"; +import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; +import { healthService } from "../services/HealthService"; +import { ResponseHandler } from "../helpers/ResponseHandler"; + +export const druidProxyRouter = express.Router(); + +// Send a 410 Gone response to all V1 API calls +druidProxyRouter.all("/datasets/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all("/dataset/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all("/datasources/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all("/data/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all("/template/v1/*", ResponseHandler.goneResponse) + +// Druid Proxy APIs for Metabase integration +druidProxyRouter.post(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative) +druidProxyRouter.get(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet) +druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) +druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) +druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file diff --git a/api-service/src/v2/routes/metricRouter.ts b/api-service/src/routes/MetricRouter.ts similarity index 56% rename from api-service/src/v2/routes/metricRouter.ts rename to api-service/src/routes/MetricRouter.ts index 92ce978c..3becdd79 100644 --- a/api-service/src/v2/routes/metricRouter.ts +++ b/api-service/src/routes/MetricRouter.ts @@ -1,6 +1,6 @@ import express from "express"; import { metricsScrapeHandler } from "../metrics/prometheus"; -export const router = express.Router(); +export const metricRouter = express.Router(); //Scrape metrics to prometheus -router.get("/metrics", metricsScrapeHandler) \ No newline at end of file +metricRouter.get("/metrics", metricsScrapeHandler) \ No newline at end of file diff --git a/api-service/src/v2/routes/Router.ts b/api-service/src/routes/Router.ts similarity index 99% rename from api-service/src/v2/routes/Router.ts rename to api-service/src/routes/Router.ts index c1df4aed..c2e6ad33 100644 --- a/api-service/src/v2/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -7,7 +7,6 @@ import DatasetRead from "../controllers/DatasetRead/DatasetRead"; import DatasetList from "../controllers/DatasetList/DatasetList" import { dataExhaust } from "../controllers/DataExhaust/DataExhaustController"; import { onRequest } from "../metrics/prometheus/helpers"; - import { Entity } from "../types/MetricModel"; import { createQueryTemplate } from "../controllers/CreateQueryTemplate/CreateTemplateController"; import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; @@ -43,5 +42,4 @@ router.post("/datasets/status-transition", setDataToRequestObject("api.datasets. router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); - +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file diff --git a/api-service/src/v2/services/CipherService.ts b/api-service/src/services/CipherService.ts similarity index 100% rename from api-service/src/v2/services/CipherService.ts rename to api-service/src/services/CipherService.ts diff --git a/api-service/src/v2/services/CloudServices/AWSStorageService.ts b/api-service/src/services/CloudServices/AWSStorageService.ts similarity index 100% rename from api-service/src/v2/services/CloudServices/AWSStorageService.ts rename to api-service/src/services/CloudServices/AWSStorageService.ts diff --git a/api-service/src/v2/services/CloudServices/AzureStorageService.ts b/api-service/src/services/CloudServices/AzureStorageService.ts similarity index 100% rename from api-service/src/v2/services/CloudServices/AzureStorageService.ts rename to api-service/src/services/CloudServices/AzureStorageService.ts diff --git a/api-service/src/v2/services/CloudServices/GCPStorageService.ts b/api-service/src/services/CloudServices/GCPStorageService.ts similarity index 100% rename from api-service/src/v2/services/CloudServices/GCPStorageService.ts rename to api-service/src/services/CloudServices/GCPStorageService.ts diff --git a/api-service/src/v2/services/CloudServices/index.ts b/api-service/src/services/CloudServices/index.ts similarity index 100% rename from api-service/src/v2/services/CloudServices/index.ts rename to api-service/src/services/CloudServices/index.ts diff --git a/api-service/src/v2/services/CloudServices/types.ts b/api-service/src/services/CloudServices/types.ts similarity index 100% rename from api-service/src/v2/services/CloudServices/types.ts rename to api-service/src/services/CloudServices/types.ts diff --git a/api-service/src/v2/services/DatasetService.ts b/api-service/src/services/DatasetService.ts similarity index 100% rename from api-service/src/v2/services/DatasetService.ts rename to api-service/src/services/DatasetService.ts diff --git a/api-service/src/v2/services/DatasetSourceConfigService.ts b/api-service/src/services/DatasetSourceConfigService.ts similarity index 100% rename from api-service/src/v2/services/DatasetSourceConfigService.ts rename to api-service/src/services/DatasetSourceConfigService.ts diff --git a/api-service/src/v2/services/DatasourceService.ts b/api-service/src/services/DatasourceService.ts similarity index 100% rename from api-service/src/v2/services/DatasourceService.ts rename to api-service/src/services/DatasourceService.ts diff --git a/api-service/src/services/HealthService.ts b/api-service/src/services/HealthService.ts new file mode 100644 index 00000000..273ceb00 --- /dev/null +++ b/api-service/src/services/HealthService.ts @@ -0,0 +1,20 @@ +import { Request, Response, NextFunction } from "express" +import { ResponseHandler } from "../helpers/ResponseHandler" + +class HealthService { + + async checkDruidHealth(req: Request, res: Response, next: NextFunction) { + ResponseHandler.successResponse(req, res, { status: 200, data: {} }) + } + + async checkKafkaHealth() : Promise { + return true + } + + async checkPostgresHealth() : Promise { + return true + } + +} + +export const healthService = new HealthService() \ No newline at end of file diff --git a/api-service/src/v2/services/QueryTemplateService.ts b/api-service/src/services/QueryTemplateService.ts similarity index 100% rename from api-service/src/v2/services/QueryTemplateService.ts rename to api-service/src/services/QueryTemplateService.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts b/api-service/src/services/SchemaGenerateService/ConfigSuggester.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/ConfigSuggester.ts rename to api-service/src/services/SchemaGenerateService/ConfigSuggester.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/Constants.json b/api-service/src/services/SchemaGenerateService/Constants.json similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/Constants.json rename to api-service/src/services/SchemaGenerateService/Constants.json diff --git a/api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts b/api-service/src/services/SchemaGenerateService/DataSchemaService.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/DataSchemaService.ts rename to api-service/src/services/SchemaGenerateService/DataSchemaService.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts b/api-service/src/services/SchemaGenerateService/SchemaAnalyser.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/SchemaAnalyser.ts rename to api-service/src/services/SchemaGenerateService/SchemaAnalyser.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts b/api-service/src/services/SchemaGenerateService/SchemaArrayValidator.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/SchemaArrayValidator.ts rename to api-service/src/services/SchemaGenerateService/SchemaArrayValidator.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts b/api-service/src/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts rename to api-service/src/services/SchemaGenerateService/SchemaCardinalityAnalyser.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts b/api-service/src/services/SchemaGenerateService/SchemaGeneratorUtils.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/SchemaGeneratorUtils.ts rename to api-service/src/services/SchemaGenerateService/SchemaGeneratorUtils.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts b/api-service/src/services/SchemaGenerateService/SchemaHandler.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/SchemaHandler.ts rename to api-service/src/services/SchemaGenerateService/SchemaHandler.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaMapping.json b/api-service/src/services/SchemaGenerateService/SchemaMapping.json similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/SchemaMapping.json rename to api-service/src/services/SchemaGenerateService/SchemaMapping.json diff --git a/api-service/src/v2/services/SchemaGenerateService/SchemaMerger.ts b/api-service/src/services/SchemaGenerateService/SchemaMerger.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/SchemaMerger.ts rename to api-service/src/services/SchemaGenerateService/SchemaMerger.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts b/api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/SuggestionTemplate.ts rename to api-service/src/services/SchemaGenerateService/SuggestionTemplate.ts diff --git a/api-service/src/v2/services/SchemaGenerateService/Template.ts b/api-service/src/services/SchemaGenerateService/Template.ts similarity index 100% rename from api-service/src/v2/services/SchemaGenerateService/Template.ts rename to api-service/src/services/SchemaGenerateService/Template.ts diff --git a/api-service/src/v2/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts similarity index 100% rename from api-service/src/v2/services/TableGenerator.ts rename to api-service/src/services/TableGenerator.ts diff --git a/api-service/src/v2/services/ValidationService.ts b/api-service/src/services/ValidationService.ts similarity index 100% rename from api-service/src/v2/services/ValidationService.ts rename to api-service/src/services/ValidationService.ts diff --git a/api-service/src/v1/services/WrapperService.ts b/api-service/src/services/WrapperService.ts similarity index 98% rename from api-service/src/v1/services/WrapperService.ts rename to api-service/src/services/WrapperService.ts index 37743d5f..061b2b14 100644 --- a/api-service/src/v1/services/WrapperService.ts +++ b/api-service/src/services/WrapperService.ts @@ -5,7 +5,8 @@ import { config } from "../configs/Config"; import { ResponseHandler } from "../helpers/ResponseHandler"; import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -export class WrapperService { +class WrapperService { + private errorHandler: ErrorResponseHandler; constructor() { this.errorHandler = new ErrorResponseHandler("WrapperService"); @@ -104,3 +105,5 @@ export class WrapperService { } } + +export const wrapperService = new WrapperService() \ No newline at end of file diff --git a/api-service/src/v2/services/telemetry.ts b/api-service/src/services/telemetry.ts similarity index 100% rename from api-service/src/v2/services/telemetry.ts rename to api-service/src/services/telemetry.ts diff --git a/api-service/src/v1/data/telemetryActions.ts b/api-service/src/telemetry/telemetryActions.ts similarity index 100% rename from api-service/src/v1/data/telemetryActions.ts rename to api-service/src/telemetry/telemetryActions.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts rename to api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DataIngestTest/Fixtures.ts b/api-service/src/tests/DatasetManagement/DataIngestTest/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DataIngestTest/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DataIngestTest/Fixtures.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts rename to api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DataOutTest/Fixtures.ts b/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DataOutTest/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetCreate/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts b/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetList/DatasetList.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetList/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetList/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DatasetList/Fixtures.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetRead/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts diff --git a/api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/DatasetUpdate/Fixtures.ts rename to api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts diff --git a/api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts b/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts rename to api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts diff --git a/api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts b/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts similarity index 100% rename from api-service/src/v2/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts rename to api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts diff --git a/api-service/src/v2/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts b/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts rename to api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts diff --git a/api-service/src/v2/tests/QueryTemplates/CreateTemplate/Fixtures.ts b/api-service/src/tests/QueryTemplates/CreateTemplate/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/CreateTemplate/Fixtures.ts rename to api-service/src/tests/QueryTemplates/CreateTemplate/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts b/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts rename to api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts diff --git a/api-service/src/v2/tests/QueryTemplates/ListTemplates/Fixtures.ts b/api-service/src/tests/QueryTemplates/ListTemplates/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/ListTemplates/Fixtures.ts rename to api-service/src/tests/QueryTemplates/ListTemplates/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts b/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts rename to api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts diff --git a/api-service/src/v2/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts b/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts rename to api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts diff --git a/api-service/src/v2/tests/QueryTemplates/TemplateQuerying/Fixtures.ts b/api-service/src/tests/QueryTemplates/TemplateQuerying/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/TemplateQuerying/Fixtures.ts rename to api-service/src/tests/QueryTemplates/TemplateQuerying/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts rename to api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts diff --git a/api-service/src/v2/tests/QueryTemplates/UpdateTemplate/Fixtures.ts b/api-service/src/tests/QueryTemplates/UpdateTemplate/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/UpdateTemplate/Fixtures.ts rename to api-service/src/tests/QueryTemplates/UpdateTemplate/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts b/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts similarity index 100% rename from api-service/src/v2/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts rename to api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts diff --git a/api-service/src/v2/tests/QueryWrapper/SqlWrapper/Fixtures.ts b/api-service/src/tests/QueryWrapper/SqlWrapper/Fixtures.ts similarity index 100% rename from api-service/src/v2/tests/QueryWrapper/SqlWrapper/Fixtures.ts rename to api-service/src/tests/QueryWrapper/SqlWrapper/Fixtures.ts diff --git a/api-service/src/v2/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts b/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts similarity index 100% rename from api-service/src/v2/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts rename to api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts diff --git a/api-service/src/v2/types/ConfigModels.ts b/api-service/src/types/ConfigModels.ts similarity index 100% rename from api-service/src/v2/types/ConfigModels.ts rename to api-service/src/types/ConfigModels.ts diff --git a/api-service/src/v1/models/ConnectionModels.ts b/api-service/src/types/ConnectionModels.ts similarity index 100% rename from api-service/src/v1/models/ConnectionModels.ts rename to api-service/src/types/ConnectionModels.ts diff --git a/api-service/src/v2/types/DatasetModels.ts b/api-service/src/types/DatasetModels.ts similarity index 100% rename from api-service/src/v2/types/DatasetModels.ts rename to api-service/src/types/DatasetModels.ts diff --git a/api-service/src/v1/models/ExhaustModels.ts b/api-service/src/types/ExhaustModels.ts similarity index 100% rename from api-service/src/v1/models/ExhaustModels.ts rename to api-service/src/types/ExhaustModels.ts diff --git a/api-service/src/v2/types/IngestionModels.ts b/api-service/src/types/IngestionModels.ts similarity index 100% rename from api-service/src/v2/types/IngestionModels.ts rename to api-service/src/types/IngestionModels.ts diff --git a/api-service/src/v2/types/MetricModel.ts b/api-service/src/types/MetricModel.ts similarity index 90% rename from api-service/src/v2/types/MetricModel.ts rename to api-service/src/types/MetricModel.ts index 7d63c191..ed68d0f9 100644 --- a/api-service/src/v2/types/MetricModel.ts +++ b/api-service/src/types/MetricModel.ts @@ -14,5 +14,5 @@ export interface Metric { } export enum Entity { - Data_in = "data-in", Data_out = "data-out", Management = "management" + Data_in = "data-in", Data_out = "data-out", Management = "management", DruidProxy = "druid-proxy" } \ No newline at end of file diff --git a/api-service/src/v2/types/ObsrvError.ts b/api-service/src/types/ObsrvError.ts similarity index 100% rename from api-service/src/v2/types/ObsrvError.ts rename to api-service/src/types/ObsrvError.ts diff --git a/api-service/src/v1/models/QueryModels.ts b/api-service/src/types/QueryModels.ts similarity index 100% rename from api-service/src/v1/models/QueryModels.ts rename to api-service/src/types/QueryModels.ts diff --git a/api-service/src/v2/types/ResponseModel.ts b/api-service/src/types/ResponseModel.ts similarity index 100% rename from api-service/src/v2/types/ResponseModel.ts rename to api-service/src/types/ResponseModel.ts diff --git a/api-service/src/v2/types/SampleURLModel.ts b/api-service/src/types/SampleURLModel.ts similarity index 100% rename from api-service/src/v2/types/SampleURLModel.ts rename to api-service/src/types/SampleURLModel.ts diff --git a/api-service/src/v2/types/SchemaModel.ts b/api-service/src/types/SchemaModel.ts similarity index 100% rename from api-service/src/v2/types/SchemaModel.ts rename to api-service/src/types/SchemaModel.ts diff --git a/api-service/src/v1/models/ValidationModels.ts b/api-service/src/types/ValidationModels.ts similarity index 100% rename from api-service/src/v1/models/ValidationModels.ts rename to api-service/src/types/ValidationModels.ts diff --git a/api-service/src/v2/utils/common.ts b/api-service/src/utils/common.ts similarity index 100% rename from api-service/src/v2/utils/common.ts rename to api-service/src/utils/common.ts diff --git a/api-service/src/v1/configs/Config.ts b/api-service/src/v1/configs/Config.ts deleted file mode 100644 index ca9834a1..00000000 --- a/api-service/src/v1/configs/Config.ts +++ /dev/null @@ -1,109 +0,0 @@ -// These configurations provide settings and values for various aspects of dataset management, data ingestion, and table configurations in a system. - -const env = process.env.system_env || "local" - -export const config = { - "env": env, - "api_port": process.env.api_port || 3000, - "body_parser_limit": process.env.body_parser_limit || "100mb", - "version": "1.0", - "query_api": { - "druid": { - "queryType": "realtime", - "host": process.env.druid_host || "http://localhost", - "port": process.env.druid_port || 8888, - "sql_query_path": "/druid/v2/sql/", - "native_query_path": "/druid/v2", - "list_datasources_path": "/druid/v2/datasources", - "submit_ingestion": "druid/indexer/v1/supervisor" - }, - "lakehouse": { - "queryType": "datalake", - "host": process.env.lakehouse_host || "http://localhost", - "port": process.env.lakehouse_port || 8080, - "catalog": process.env.lakehouse_catalog || "hudi_connector", - "schema": process.env.lakehouse_schema || "obsrv", - "default_user": process.env.lakehouse_default_user || "admin" - } - }, - "db_connector_config": { - client: "postgresql", - connection: { - host: process.env.postgres_host || 'localhost', - port: process.env.postgres_port || 5432, - database: process.env.postgres_database || 'obsrv', - user: process.env.postgres_username || 'postgres', - password: process.env.postgres_password || 'postgres', - } - }, - "telemetry_service_config": { - level: process.env.telemetry_log_level || 'info', - localStorageEnabled: process.env.telemetry_local_storage_enabled || 'true', - dispatcher: process.env.telemetry_local_storage_type || 'kafka', - telemetryProxyEnabled: process.env.telemetry_proxy_enabled, - proxyURL: process.env.telemetry_proxy_url, - proxyAuthKey: process.env.telemetry_proxy_auth_key, - compression_type: process.env.telemetry_kafka_compression || 'none', - filename: process.env.telemetry_file_filename || 'telemetry-%DATE%.log', - maxsize: process.env.telemetry_file_maxsize || 10485760, - maxFiles: process.env.telemetry_file_maxfiles || '100', - "kafka": { // The default Kafka configuration includes essential parameters such as broker IP addresses and other configuration options. - "config": { - "brokers": [`${process.env.kafka_host || 'localhost'}:${process.env.kafka_port || 9092}`], - "clientId": process.env.client_id || "obsrv-apis", - "retry": { - "initialRetryTime": process.env.kafka_initial_retry_time ? parseInt(process.env.kafka_initial_retry_time) : 3000, - "retries": process.env.kafka_retries ? parseInt(process.env.kafka_retries) : 1 - }, - "connectionTimeout": process.env.kafka_connection_timeout ? parseInt(process.env.kafka_connection_timeout) : 5000 - }, - "topics": { // Default Kafka topics depend on type of dataset. - "createDataset": `${process.env.system_env || 'local'}.ingest`, - "createMasterDataset": `${process.env.system_env || 'local'}.masterdata.ingest` - } - } - }, - "dataset_types": { - normalDataset: "dataset", - masterDataset: "master-dataset" - }, - "datasource_storage_types": { - druid: "druid", - datalake: "datalake" - }, - "redis_config": { - "redis_host": process.env.redis_host || 'localhost', - "redis_port": process.env.redis_port || 6379 - }, - "exclude_datasource_validation": process.env.exclude_datasource_validation ? process.env.exclude_datasource_validation.split(",") : ["system-stats", "failed-events-summary", "masterdata-system-stats", "system-events"], // list of datasource names to skip validation while calling query API - "telemetry_dataset": process.env.telemetry_dataset || `${env}.system.telemetry.events`, - "table_names": { // Names of all tables available for CRUD operations - "datasets": "datasets", - "datasources": "datasources", - "datasetSourceConfig": "dataset_source_config" - }, - "table_config": { // This object defines the configuration for each table. - "datasets": { - "primary_key": "id", - "references": [] - }, - "datasources": { - "primary_key": "id", - "references": [] - }, - "dataset_source_config": { - "primary_key": "id", - "references": [] - } - }, - "exhaust_config": { - "cloud_storage_provider": process.env.cloud_storage_provider || "aws", // Supported providers - AWS, GCP, Azure - "cloud_storage_region": process.env.cloud_storage_region || "", // Region for the cloud provider storage - "cloud_storage_config": process.env.cloud_storage_config ? JSON.parse(process.env.cloud_storage_config) : {}, // Respective credentials object for cloud provider. Optional if service account provided - "container": process.env.container || "", // Storage container/bucket name - "container_prefix": process.env.container_prefix || "", // Path to the folder inside container/bucket. Empty if data at root level - "storage_url_expiry": process.env.storage_url_expiry ? parseInt(process.env.storage_url_expiry) : 3600, // in seconds, Default 1hr of expiry for Signed URLs. - "maxQueryDateRange": process.env.exhaust_query_range ? parseInt(process.env.exhaust_query_range) : 31, // in days. Defines the maximum no. of days the files can be fetched - "exclude_exhaust_types": process.env.exclude_exhaust_types ? process.env.exclude_exhaust_types.split(",") : ["system-stats", "masterdata-system-stats", "system-events",] // list of folder type names to skip exhaust service - }, -} diff --git a/api-service/src/v1/configs/Extensions.ts b/api-service/src/v1/configs/Extensions.ts deleted file mode 100644 index 967e0e09..00000000 --- a/api-service/src/v1/configs/Extensions.ts +++ /dev/null @@ -1,11 +0,0 @@ - -export const extensions: Array = [ - - -] - - -interface IExtensionConfig { - id: string; - routePath: string; -} \ No newline at end of file diff --git a/api-service/src/v1/configs/RoutesConfig.ts b/api-service/src/v1/configs/RoutesConfig.ts deleted file mode 100644 index 37720b2b..00000000 --- a/api-service/src/v1/configs/RoutesConfig.ts +++ /dev/null @@ -1,181 +0,0 @@ -export const routesConfig = { - default: { - api_id: "api", - validation_schema: null, - }, - query: { - native_query: { - api_id: "native.query", - method: "post", - path: "/data/v1/query", - validation_schema: "QueryRequest.json", - }, - native_query_with_params: { - api_id: "native.query", - method: "post", - path: "/data/v1/query/:datasetId", - validation_schema: "QueryRequest.json", - }, - sql_query: { - api_id: "sql.query", - method: "post", - path: "/data/v1/sql-query", - validation_schema: "QueryRequest.json", - }, - sql_query_with_params: { - api_id: "sql.query", - method: "post", - path: "/data/v1/sql-query/:datasetId", - validation_schema: "QueryRequest.json", - }, - }, - config: { - dataset: { - save: { - api_id: "config.dataset.create", - method: "post", - path: "/datasets/v1/create", - validation_schema: "DatasetCreateReq.json", - }, - read: { - api_id: "config.dataset.read", - method: "get", - path: "/datasets/v1/get/:datasetId", - validation_schema: null, - }, - update: { - api_id: "config.dataset.update", - method: "patch", - path: "/datasets/v1/update", - validation_schema: "DatasetUpdateReq.json", - }, - list: { - api_id: "config.dataset.list", - method: "post", - path: "/datasets/v1/list", - validation_schema: "DatasetListReq.json", - }, - }, - datasource: { - save: { - api_id: "config.datasource.create", - method: "post", - path: "/datasources/v1/create", - validation_schema: "DatasourceSaveReq.json", - }, - read: { - api_id: "config.datasource.read", - method: "get", - path: "/datasources/v1/get/:datasourceId", - validation_schema: null, - }, - update: { - api_id: "config.datasource.update", - method: "patch", - path: "/datasources/v1/update", - validation_schema: "DatasourceUpdateReq.json", - }, - list: { - api_id: "config.datasource.list", - method: "post", - path: "/datasources/v1/list", - validation_schema: "DatasetListReq.json", - }, - }, - dataset_source_config: { - save: { - api_id: "config.dataset.source.config.create", - method: "post", - path: "/datasets/v1/source/config/create", - validation_schema: "DatasetSourceConfigSaveReq.json", - }, - read: { - api_id: "config.dataset.source.config.read", - method: "get", - path: "/datasets/v1/source/config/get/:datasetId", - validation_schema: null, - }, - update: { - api_id: "config.dataset.source.config.update", - method: "patch", - path: "/datasets/v1/source/config/update", - validation_schema: "DatasetSourceConfigUpdateReq.json", - }, - list: { - api_id: "config.dataset.source.config.list", - method: "post", - path: "/datasets/v1/source/config/list", - validation_schema: "DatasetListReq.json", - }, - } - - }, - data_ingest: { - api_id: "dataset.data.in", - method: "post", - path: "/data/v1/in/:datasetId", - validation_schema: "DataIngestionReq.json", - }, - tenant_ingest: { - api_id: "dataset.data.in", - method: "post", - path: "/data/tenant/in/:datasetId", - validation_schema: "DataIngestionReq.json", - }, - exhaust: { - api_id: "dataset.data.exhaust", - method: "get", - path: "/data/v1/exhaust/:datasetId", - validation_schema: "DataExhaustReq.json" - }, - prometheus: { - method: "get", - path: "/metrics", - validation_schema: null, - }, - submit_ingestion: { - api_id: "submit.ingestion", - method: "post", - path: "/data/v1/submit/ingestion", - validation_schema: "SubmitIngestionReq.json" - }, - query_wrapper: { - sql_wrapper: { - api_id: "query.wrapper.sql.query", - method: "post", - path: "/v1/sql", - }, - native_post: { - api_id: "query.wrapper.native.post", - method: "post", - path: /\/druid\/v2.*/, - }, - native_get: { - api_id: "query.wrapper.native.get", - method: "get", - path: /\/druid\/v2.*/ - }, - native_delete: { - api_id: "query.wrapper.native.delete", - method: "delete", - path: "/druid/v2/:queryId" - }, - druid_status: { - api_id: "query.wrapper.status", - method: "get", - path: "/status" - } - }, - health: { - api_id: "api.health", - method: "get", - path: "/health" - }, - schema_validator:{ - api_id: "api.schema.validate", - method: "post", - path: "/data/v1/schema/validate", - validation_schema: "SchemaValidatorReq.json" - } -} - diff --git a/api-service/src/v1/connectors/DbConnector.ts b/api-service/src/v1/connectors/DbConnector.ts deleted file mode 100644 index f3b4f067..00000000 --- a/api-service/src/v1/connectors/DbConnector.ts +++ /dev/null @@ -1,179 +0,0 @@ -import knex, { Knex } from "knex"; -import { IConnector } from "../models/DatasetModels"; -import { DbConnectorConfig } from "../models/ConnectionModels"; -import { SchemaMerger } from "../generators/SchemaMerger"; -import constants from '../resources/Constants.json' -import { config as appConfig } from "../configs/Config" -import _ from 'lodash' -import { wrapperService } from "../routes/Router"; -const schemaMerger = new SchemaMerger() - -const OP_TYPES = { - INSERT: "insert", - UPDATE: "update", - UPSERT: "upsert", - READ: "read", - LIST: "list", - DELETE: "delete", - SQL: "sql", -} -export class DbConnector implements IConnector { - public pool: Knex - private config: DbConnectorConfig - public typeToMethod = { - insert: this.insertRecord, - update: this.updateRecord, - read: this.readRecords, - upsert: this.upsertRecord, - } - public method: any - constructor(config: DbConnectorConfig) { - this.config = config; - this.pool = knex(this.config) - } - public init = () => { - this.connect() - .then(() => console.info("Database Connection Established...")) - .catch((err: Error) => console.error(`Database Connection failed: ${err.message}`)) - } - - async connect() { - await this.pool.select(1) - } - - async close() { - return await this.pool.destroy() - } - - async health() { - await this.connect() - await this.pool.select(1) - } - - execute(type: keyof typeof this.typeToMethod, property: any) { - this.method = this.typeToMethod[ type ] - return this.method(property[ "table" ], property[ "fields" ]) - } - - public async insertRecord(table: string, fields: any) { - await this.pool.transaction(async (dbTransaction) => { - await this.submit_ingestion(_.get(fields, 'ingestion_spec'), table, _.get(fields, "type")) - await dbTransaction(table).insert(fields).on('query-error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, fields); - throw {...constants.FAILED_RECORD_CREATE, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, fields); - throw {...constants.FAILED_RECORD_CREATE, "errCode": error.code} - }); - }) - } - - public async updateRecord(table: string, fields: any) { - const { filters, values } = fields - await this.pool.transaction(async (dbTransaction) => { - const currentRecord = await dbTransaction(table).select(Object.keys(values)).where(filters).first() - if (_.isUndefined(currentRecord)) { throw constants.FAILED_RECORD_UPDATE } - if (!_.isUndefined(currentRecord.tags)) { delete currentRecord.tags } - await dbTransaction(table).where(filters).update(schemaMerger.mergeSchema(currentRecord, values)).on('query-error', (error: any) => { - this.log_error(OP_TYPES.UPDATE, error, table, values); - throw {...constants.FAILED_RECORD_UPDATE, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, values); - throw {...constants.FAILED_RECORD_UPDATE, "errCode": error.code} - }); - }) - } - - public async upsertRecord(table: string, fields: any) { - const { filters, values } = fields; - const existingRecord = await this.pool(table).select().where(filters).first() - if (!_.isUndefined(existingRecord)) { - await this.pool.transaction(async (dbTransaction) => { - await this.submit_ingestion(_.get(values, 'ingestion_spec'), table, _.get(fields, "type")) - await dbTransaction(table).where(filters).update(schemaMerger.mergeSchema(existingRecord, values)).on('query-error', (error: any) => { - this.log_error(OP_TYPES.UPSERT, error, table, values); - throw {...constants.FAILED_RECORD_UPDATE, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, values); - throw constants.FAILED_RECORD_CREATE - }); - }) - } else { - await this.pool.transaction(async (dbTransaction) => { - await this.submit_ingestion(_.get(values, 'ingestion_spec'), table, _.get(fields, "type")) - await dbTransaction(table).insert(values).on('query-error', (error: any) => { - this.log_error(OP_TYPES.UPSERT, error, table, values); - throw {...constants.FAILED_RECORD_CREATE, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, values); - throw {...constants.FAILED_RECORD_CREATE, "errCode": error.code} - }); - }) - } - } - - public async readRecords(table: string, fields: any) { - const query = this.pool.from(table).select().where((builder) => { - const filters = fields.filters || {}; - if (filters.status) { - if (Array.isArray(filters.status) && filters.status.length > 0) { - builder.whereIn("status", filters.status); - } else if (filters.status.length > 0) { - builder.where("status", filters.status); - } - } - delete filters.status; - builder.where(filters).on('query-error', (error: any) => { - this.log_error(OP_TYPES.READ, error, table, filters); - throw {...constants.FAILED_RECORD_FETCH, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, filters); - throw {...constants.FAILED_RECORD_FETCH, "errCode": error.code} - }); - }) - return await query - } - - public async listRecords(table: string) { - return await this.pool.select('*').from(table).on('query-error', (error: any) => { - this.log_error(OP_TYPES.LIST, error, table, {}); - throw {...constants.FAILED_RECORD_FETCH, "errCode": error.code} - }).on('error', (error: any) => { - this.log_error(OP_TYPES.INSERT, error, table, {}); - throw {...constants.FAILED_RECORD_FETCH, "errCode": error.code} - }) - } - - private async submit_ingestion(ingestion_spec: Record, table: string, storage_type: string) { - if (appConfig.table_names.datasources === table && storage_type === appConfig.datasource_storage_types.druid) { - return await wrapperService.submitIngestion(ingestion_spec) - .catch((error: any) => { - console.error(constants.INGESTION_FAILED_ON_SAVE) - throw {...constants.FAILED_RECORD_UPDATE, "errCode": error.code} - }) - } - return - } - - public async executeSql(sql: string[]) { - let result: any[] = []; - await this.pool.transaction(async (dbTransaction) => { - for(let query of sql) { - result.push(await dbTransaction.raw(query).on('query-error', (error: any) => { - this.log_error(OP_TYPES.SQL, error, JSON.stringify(query), {}); - throw {...constants.FAILED_SQL_QUERY, "errCode": error.code}; - })); - } - }); - return result; - } - - private log_error(op_type: string, error: any, table: string, values: any) { - console.log(` - Error occured for operation ${op_type} - - Table - ${table} - Values - ${JSON.stringify(values)} - Error - ${JSON.stringify(error)} - `); - } -} diff --git a/api-service/src/v1/connectors/HttpConnector.ts b/api-service/src/v1/connectors/HttpConnector.ts deleted file mode 100644 index 26eff56a..00000000 --- a/api-service/src/v1/connectors/HttpConnector.ts +++ /dev/null @@ -1,28 +0,0 @@ -import axios, { AxiosInstance } from "axios"; -import { IConnector } from "../models/DatasetModels"; -export class HTTPConnector implements IConnector { - private url: string; - constructor(url: string) { - this.url = url - } - connect(): AxiosInstance { - return axios.create({ - baseURL: this.url, - timeout: 3000, - headers: { "Content-Type": "application/json" } - }); - } - - execute(sample: string) { - throw new Error("Method not implemented."); - } - executeSql(sql: string[]) { - throw new Error("Method not implemented."); - } - close() { - throw new Error("Method not implemented."); - } - -} - - diff --git a/api-service/src/v1/connectors/KafkaConnector.ts b/api-service/src/v1/connectors/KafkaConnector.ts deleted file mode 100644 index 3ce6f848..00000000 --- a/api-service/src/v1/connectors/KafkaConnector.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { IConnector } from "../models/DatasetModels" -const telemetryService = require('../lib/services/TelemetryService') - -export class KafkaConnector implements IConnector { - public telemetryService: any - constructor() { - this.telemetryService = telemetryService - } - async connect() { - await telemetryService.health() - - } - - async execute(req: any, res: any, topic: any) { - await this.telemetryService.dispatch(req, res, topic) - } - - async executeSql(sql: string[]) { - throw new Error("Method not implemented") - } - - close() { - throw new Error("Method not implemented") - } -} diff --git a/api-service/src/v1/generators/SchemaMerger.ts b/api-service/src/v1/generators/SchemaMerger.ts deleted file mode 100644 index 942a1f9e..00000000 --- a/api-service/src/v1/generators/SchemaMerger.ts +++ /dev/null @@ -1,7 +0,0 @@ -import * as _ from "lodash"; -export class SchemaMerger { - - mergeSchema(schema: any, schema2: any): Map { - return _.merge(schema, schema2); - } -} diff --git a/api-service/src/v1/helpers/DatasetConfigs.ts b/api-service/src/v1/helpers/DatasetConfigs.ts deleted file mode 100644 index 3f620619..00000000 --- a/api-service/src/v1/helpers/DatasetConfigs.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { dbConnector } from "../routes/Router" -import { globalCache } from "../routes/Router"; - - -function getDatasetConfigs() { - return dbConnector.listRecords('datasets') -} - -export const refreshDatasetConfigs = async () => { - try { - const data = await getDatasetConfigs(); - globalCache.set("dataset-config", data) - } - catch (err) { - console.log(err); - throw new Error("Failed to cache configs") - } -} - diff --git a/api-service/src/v1/helpers/DatasetSourceConfigs.ts b/api-service/src/v1/helpers/DatasetSourceConfigs.ts deleted file mode 100644 index af3e0832..00000000 --- a/api-service/src/v1/helpers/DatasetSourceConfigs.ts +++ /dev/null @@ -1,52 +0,0 @@ -import _ from "lodash"; -import { SchemaMerger } from "../generators/SchemaMerger" -import { defaultConfig } from '../resources/schemas/DatasetConfigDefault' -import {v4} from 'uuid' -import { DatasetStatus } from "../models/DatasetModels"; -const schemaMerger = new SchemaMerger() -export class DatasetSourceConfigs { - private id: string - private dataset_id: string - private connector_type: string - private connector_config: object - private status: DatasetStatus - private connector_stats: object - private created_by: string - private updated_by: string - private published_date: Date - constructor(payload: any) { - if (payload.id) { - this.id = payload.id - } - else { - this.id = v4() - } - this.dataset_id = payload.dataset_id - this.connector_type = payload.connector_type - this.connector_config = payload.connector_config - this.status = payload.status - this.connector_stats = payload.connector_stats - this.created_by = payload.created_by - this.updated_by = payload.updated_by - this.published_date = payload.published_date - } - - public getValues() { - return Object.assign(this.removeNullValues({ id: this.id, dataset_id: this.dataset_id, connector_type: this.connector_type, connector_config: this.connector_config, status: this.status, connector_stats: this.connector_stats, created_by: this.created_by, updated_by: this.updated_by, published_date: this.published_date}), { "updated_date": new Date }) - } - - public setValues() { - return schemaMerger.mergeSchema(this.getDefaults(), this.getValues()) - } - - public removeNullValues(payload: any) { - Object.keys(payload).map((value) => { - if (_.isEmpty(payload[value])) delete payload[value] - }) - return payload - } - - public getDefaults() { - return {...defaultConfig.sourceConfig} - } -} diff --git a/api-service/src/v1/helpers/Datasets.ts b/api-service/src/v1/helpers/Datasets.ts deleted file mode 100644 index 07d4daa3..00000000 --- a/api-service/src/v1/helpers/Datasets.ts +++ /dev/null @@ -1,97 +0,0 @@ -import _ from 'lodash' -import { ValidationConfig, ExtractionConfig, DedupConfig, DenormConfig, RouterConfig, DatasetConfig } from '../models/ConfigModels' -import { defaultConfig } from '../resources/schemas/DatasetConfigDefault' -import { SchemaMerger } from '../generators/SchemaMerger' -import { config } from '../configs/Config' -import { DatasetStatus } from '../models/DatasetModels' -import constants from "../resources/Constants.json"; - -let schemaMerger = new SchemaMerger() -export class Datasets { - private id: string - private type: string - private name: string - private dataset_id: string - private validation_config: object - private extraction_config: object - private dedup_config: object - private data_schema: object - private router_config: object - private denorm_config: object - private dataset_config: object - private tags: any - private status: DatasetStatus - private created_by: string - private updated_by: string - private published_date: Date - private api_version: string - private version_key: string - constructor(payload: any) { - if (payload.id) { - this.id = payload.id - } - else { - this.id = payload.dataset_id - } - this.dataset_id = payload.dataset_id - this.type = payload.type - this.name = payload.name - this.validation_config = payload.validation_config - this.extraction_config = payload.extraction_config - this.dedup_config = payload.dedup_config - this.data_schema = payload.data_schema - this.router_config = payload.router_config - this.denorm_config = payload.denorm_config - this.dataset_config = payload.dataset_config - this.tags = payload.tags - this.status = payload.status - this.created_by = payload.created_by - this.updated_by = payload.updated_by - this.published_date = payload.published_date - this.api_version = "v1" - this.version_key = payload.version_key || "1" - } - - public getValues() { - this.validateDenormConfig(); - return Object.assign(this.removeNullValues({ id: this.id, dataset_id: this.dataset_id, type: this.type, name: this.name, validation_config: this.validation_config, extraction_config: this.extraction_config, dedup_config: this.dedup_config, data_schema: this.data_schema, router_config: this.router_config, denorm_config: this.denorm_config, dataset_config: this.dataset_config, tags: this.tags, status: this.status, api_version: this.api_version, version_key: this.version_key, created_by: this.created_by, updated_by: this.updated_by, published_date: this.published_date }), { "updated_date": new Date }) - } - - public setValues() { - this.validateDenormConfig(); - return schemaMerger.mergeSchema(this.getDefaults(), this.getValues()) - } - - public removeNullValues(payload: any) { - Object.keys(payload).map((value) => { - if (_.isEmpty(payload[value])) delete payload[value] - }) - return payload - } - - public getDefaults() { - if (this.type == config.dataset_types.masterDataset) { - return {...defaultConfig.master} - } - else { - return {...defaultConfig.dataset} - } - } - - private validateDenormConfig() { - if (this.denorm_config && _.has(this.denorm_config, 'denorm_fields')) { - let duplicatesExist = false; - let denormFields: any = _.get(this.denorm_config, 'denorm_fields', []); - denormFields = _.map(denormFields, (denormField: Record) => _.get(denormField, 'denorm_out_field')); - denormFields.map( - (denormField: string | number) => { - if(_.indexOf(denormFields, denormField) !== _.lastIndexOf(denormFields, denormField)) - duplicatesExist = true; - } - ); - if(duplicatesExist) { - throw constants.DUPLICATE_DENORM_FIELD; - } - } - } -} diff --git a/api-service/src/v1/helpers/Datasources.ts b/api-service/src/v1/helpers/Datasources.ts deleted file mode 100644 index 2a3d1051..00000000 --- a/api-service/src/v1/helpers/Datasources.ts +++ /dev/null @@ -1,65 +0,0 @@ -import _ from 'lodash' -import configDefault from '../resources/schemas/DatasourceConfigDefault.json' -import { SchemaMerger } from '../generators/SchemaMerger' -import { DatasetStatus } from '../models/DatasetModels' -let schemaMerger = new SchemaMerger - -export class Datasources { - private id: string - private dataset_id: string - private ingestion_spec: object - private type: string - private datasource: string - private datasource_ref: string - private retention_period: object - private archival_policy: object - private purge_policy: object - private backup_config: object - private status: DatasetStatus - private created_by: string - private updated_by: string - private version: string - private published_date: Date - private metadata: object - - constructor(payload: any) { - if (payload.id) { - this.id = payload.id - } - else { - this.id = payload.dataset_id + '_' + payload.datasource - } - this.dataset_id = payload.dataset_id - this.ingestion_spec = payload.ingestion_spec - this.type = payload.type - this.datasource = payload.datasource - this.datasource_ref = payload.datasource_ref - this.retention_period = payload.retentionPeriod - this.archival_policy = payload.archivalPolicy - this.purge_policy = payload.purgePolicy - this.backup_config = payload.backup_config - this.status = payload.status - this.version = payload.version - this.created_by = payload.created_by - this.updated_by = payload.updated_by - this.published_date = payload.published_date - this.metadata = payload.metadata - } - public getValues() { - return Object.assign(this.removeNullValues({ id: this.id, dataset_id: this.dataset_id, ingestion_spec: this.ingestion_spec, type: this.type, datasource: this.datasource, datasource_ref: this.datasource_ref, retention_period: this.retention_period, archival_policy: this.archival_policy, purge_policy: this.purge_policy, backup_config: this.backup_config, status: this.status, version: this.version, created_by: this.created_by, updated_by: this.updated_by, published_date: this.published_date, metadata: this.metadata }), { "updated_date": new Date }) - } - - public setValues() { - return schemaMerger.mergeSchema(this.getDefaults(), this.getValues()); - } - - public removeNullValues(payload: any) { - Object.keys(payload).map((value) => { - if (_.isEmpty(payload[value])) delete payload[value] - }) - return payload - } - public getDefaults() { - return {...configDefault} - } -} diff --git a/api-service/src/v1/helpers/DbUtil.ts b/api-service/src/v1/helpers/DbUtil.ts deleted file mode 100644 index f9c95091..00000000 --- a/api-service/src/v1/helpers/DbUtil.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import constants from "../resources/Constants.json"; -import { config } from "../configs/Config"; -import _ from "lodash"; -import { IConnector } from "../models/DatasetModels"; - -export class DbUtil { - private dbConnector: IConnector - private table: string - constructor(dbConnector: IConnector, table: string) { - this.dbConnector = dbConnector - this.table = table - } - public save = async (req: Request, res: Response, next: NextFunction, fields: any) => { - const filters = this.getFilters(this.table, fields) - const fetchedRecord = await this.dbConnector.execute("read", { table: this.table, fields: { "filters": filters } }) - if (!_.isEmpty(fetchedRecord)) { throw constants.DUPLICATE_RECORD } - await this.dbConnector.execute("insert", { "table": this.table, "fields": fields }) - .then(() => { - console.log(constants.RECORD_SAVED) - ResponseHandler.successResponse(req, res, { status: 200, data: Object.assign({ message: constants.RECORD_SAVED }, filters) }); - }) - } - public update = async (req: Request, res: Response, next: NextFunction, fields: any) => { - const filters = this.getFilters(this.table, fields) - await this.dbConnector.execute("update", { "table": this.table, "fields": { "filters": filters, "values": fields } }) - .then(() => { - console.log(constants.RECORD_UPDATED) - ResponseHandler.successResponse(req, res, { status: 200, data: Object.assign({ message: constants.RECORD_UPDATED }, filters) }); - }) - } - public upsert = async (req: Request, res: Response, next: NextFunction, fields: any) => { - const filters = this.getFilters(this.table, fields) - await this.dbConnector.execute("upsert", { "table": this.table, "fields": { "filters": filters, "values": fields } }) - .then(() => { - console.log(constants.RECORD_UPDATED) - ResponseHandler.successResponse(req, res, { status: 200, data: Object.assign({ message: constants.RECORD_UPDATED }, filters) }); - }) - } - public read = async (req: Request, res: Response, next: NextFunction, fields: any) => { - const filters = this.getFilters(this.table, fields) - await this.dbConnector.execute("read", { "table": this.table, "fields": { "filters": filters } }) - .then((data: any[]) => { - !_.isEmpty(data) ? ResponseHandler.successResponse(req, res, { status: 200, data: _.first(data) }) : (() => { throw constants.RECORD_NOT_FOUND; })() - }) - } - public list = async (req: Request, res: Response, next: NextFunction, fields: any) => { - await this.dbConnector.execute("read", { "table": this.table, "fields": fields }) - .then((data: any) => { - ResponseHandler.successResponse(req, res, { status: 200, data: data }); - }) - } - private getFilters = (table: string, fields: any) => { - let filters: any = {} - const table_config: any = config["table_config"][table as keyof typeof config.table_config] - filters[table_config["primary_key"]] = fields[table_config["primary_key"]] - return filters - } -} diff --git a/api-service/src/v1/helpers/ErrorResponseHandler.ts b/api-service/src/v1/helpers/ErrorResponseHandler.ts deleted file mode 100644 index f680c3e2..00000000 --- a/api-service/src/v1/helpers/ErrorResponseHandler.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { NextFunction, Request, Response } from "express"; -import httpStatus from "http-status"; -import { setAuditState } from "../services/telemetry"; - -export class ErrorResponseHandler { - private serviceName: string; - constructor(serviceName: string) { - this.serviceName = serviceName; - } - public handleError(req: Request, res: Response, next: NextFunction, error: any, audit: boolean = true): any { - console.error("Error in " + this.serviceName) - console.error(JSON.stringify({ - "ts": Date.now(), - "body": req.body, - "headers": req.headers, - "url": req.url, - "error": { - "message": error?.message, - "stack": error?.stack, - "data": error?.data, - "code": error?.code, - "error": error, - } - })); - if(audit) setAuditState("failed", req); - next({ - statusCode: error.status || httpStatus.INTERNAL_SERVER_ERROR, - message: error.message, - errCode: httpStatus[`${error.status}_NAME`] || httpStatus["500_NAME"], - }); - } -}; diff --git a/api-service/src/v1/helpers/LakehouseUtil.ts b/api-service/src/v1/helpers/LakehouseUtil.ts deleted file mode 100644 index cb5e2026..00000000 --- a/api-service/src/v1/helpers/LakehouseUtil.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { Trino, BasicAuth } from 'trino-client'; -import _ from 'lodash'; -import { config } from '../configs/Config'; - -const trino: Trino = Trino.create({ - server: `${config.query_api.lakehouse.host}:${config.query_api.lakehouse.port}`, - catalog: config.query_api.lakehouse.catalog, - schema: config.query_api.lakehouse.schema, - auth: new BasicAuth(config.query_api.lakehouse.default_user), -}); - - -const getFormattedData = (data: any[], columnData: any[]) => { - const formattedData: any[] = []; - for (let i = 0; i < data.length; i++) { - const row = data[ i ]; - const jsonRow: any = {}; - for (let j = 0; j < row.length; j++) { - // assign column only if doesn't start with _hoodie_ - const colName = columnData[ j ]; - if (_.startsWith(colName, "_hoodie_")) { - continue; - } - jsonRow[ colName ] = row[ j ]; - } - formattedData.push(jsonRow); - } - return formattedData; -} - - -export const executeLakehouseQuery = async (query: string) => { - const iter = await trino.query(query); - let queryResult: any = [] - for await (let data of iter) { - if(!_.isEmpty(data.error)){ - throw { - status: 400, - message: data.error.message.replace(/line.*: /, ''), - code: "BAD_REQUEST" - } - } - queryResult = [ ...queryResult, ...(data?.data?.length ? data.data : []) ] - } - let columns = await iter.map((r: any) => r.columns ?? []).next(); - let finalColumns = columns.value.map((column: any) => { - return column.name; - }); - const formattedData = getFormattedData(queryResult, finalColumns); - return formattedData -} \ No newline at end of file diff --git a/api-service/src/v1/helpers/ResponseHandler.ts b/api-service/src/v1/helpers/ResponseHandler.ts deleted file mode 100644 index 89b21880..00000000 --- a/api-service/src/v1/helpers/ResponseHandler.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { ErrorRequestHandler, NextFunction, Request, Response } from "express"; -import httpStatus from "http-status"; -import { IResponse, Result } from "../models/DatasetModels"; -import constants from "../resources/Constants.json"; -import { routesConfig } from "../configs/RoutesConfig"; -import { onFailure, onSuccess } from "./../../v2/metrics/prometheus/helpers"; -type extendedErrorRequestHandler = ErrorRequestHandler & { - statusCode: number; - message: string; - errCode: string; - id?: string; -}; - -const ResponseHandler = { - successResponse: (req: Request, res: Response, result: Result) => { - const { entity } = req as any; - res.status(result.status || 200).json(ResponseHandler.refactorResponse({ id: (req as any).id, result: result.data })); - entity && onSuccess(req, res) - }, - - routeNotFound: (req: Request, res: Response, next: NextFunction) => { - next({ statusCode: httpStatus.NOT_FOUND, message: constants.ERROR_MESSAGE.ROUTE_NOT_FOUND, errCode: httpStatus["404_NAME"] }); - }, - - refactorResponse: ({ id = routesConfig.default.api_id, ver = "v1", params = { status: constants.STATUS.SUCCESS, errmsg: "" }, responseCode = httpStatus["200_NAME"], result = {} }): IResponse => { - return { id, ver, ts: Date.now(), params, responseCode, result } - }, - - errorResponse: (error: extendedErrorRequestHandler, req: Request, res: Response, next: NextFunction) => { - const { statusCode, message, errCode } = error; - const { id, entity } = req as any; - res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json(ResponseHandler.refactorResponse({ id: id, params: { status: constants.STATUS.FAILURE, errmsg: message, }, responseCode: errCode || httpStatus["500_NAME"] })); - entity && onFailure(req, res) - }, - - setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { - (req as any).id = id; - next(); - }, - - flatResponse: (req: Request, res: Response, result: Result) => { - const { entity } = req as any; - entity && onSuccess(req, res) - res.status(result.status).send(result.data); - }, - - goneResponse: (req: Request, res: Response) => { - const { id, entity } = req as any; - res.status(httpStatus.GONE).json({ id: id, ver: "v1", ts: Date.now(), params: { status: constants.STATUS.FAILURE, errmsg: "v1 APIs have been replace by /v2 APIs. Please refer to this link for more information" }, responseCode: httpStatus["410_NAME"] }) - } -} - -export { ResponseHandler }; diff --git a/api-service/src/v1/helpers/ValidationService.ts b/api-service/src/v1/helpers/ValidationService.ts deleted file mode 100644 index 57d6d35f..00000000 --- a/api-service/src/v1/helpers/ValidationService.ts +++ /dev/null @@ -1,12 +0,0 @@ -import Ajv from "ajv"; -const validator = new Ajv({ strict: false }); - -export const schemaValidation = (payload: Record, schema: Record): Record => { - const isValid = validator.validate(schema, payload) - if (!isValid) { - const error: any = validator.errors; - const errorMessage = error[0]?.schemaPath?.replace("/", "") + " " + error[0]?.message || "Invalid Request Body"; - return { isValid, message: errorMessage } - } - return { isValid, message: "success" } -} \ No newline at end of file diff --git a/api-service/src/v1/lib/client-cloud-services/AWSStorageService.js b/api-service/src/v1/lib/client-cloud-services/AWSStorageService.js deleted file mode 100644 index d8661ec4..00000000 --- a/api-service/src/v1/lib/client-cloud-services/AWSStorageService.js +++ /dev/null @@ -1,475 +0,0 @@ -/** - * @file - AWS Storage Service - * @exports - `AWSStorageService` - * @since - 5.0.1 - * @version - 1.0.0 - * @implements - BaseStorageService - * @see {@link https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html | X-Amz-Credential} - * @see {@link https://docs.aws.amazon.com/directconnect/latest/APIReference/CommonParameters.html#CommonParameters-X-Amz-Credential | X-Amz-Credential} - */ - -const BaseStorageService = require("./BaseStorageService"); -const { logger } = require("@project-sunbird/logger"); -const _ = require("lodash"); -const dateFormat = require("dateformat"); -const uuidv1 = require("uuid/v1"); -const async = require("async"); -const storageLogger = require("./storageLogger"); -const { getSignedUrl } = require("@aws-sdk/s3-request-presigner"); -const { S3Client, GetObjectCommand, HeadObjectCommand, PutObjectCommand, ListObjectsV2Command, } = require("@aws-sdk/client-s3"); -const { fromInstanceMetadata } = require("@aws-sdk/credential-providers"); -const { Upload } = require("@aws-sdk/lib-storage"); -const multiparty = require("multiparty"); -const moment = require("moment"); -const { config: globalConfig } = require("../../configs/Config"); -const { getFileKey } = require("../../utils/common"); - -class AWSStorageService extends BaseStorageService { - constructor(config) { - super(); - if (_.get(config, "identity") && _.get(config, "credential") && _.get(config, "region")) { - process.env.AWS_ACCESS_KEY_ID = _.get(config, "identity"); - process.env.AWS_SECRET_ACCESS_KEY = _.get(config, "credential"); - const region = _.get(config, "region").toString(); - this.client = new S3Client({ region }); - } else { - const region = globalConfig.exhaust_config.exhaust_region || "us-east-2"; - process.env.AWS_REGION = region; - const s3Client = new S3Client({ - region, - }); - this.client = s3Client; - } - } - - /** - * @description - Function to generate AWS command for an operation - * @param {string} bucketName - AWS bucket name - * @param {string} fileToGet - AWS File to fetch - * @param {string} prefix - `Optional` - Prefix for file path - * @returns - AWS Command to be executed by SDK - */ - getAWSCommand(bucketName, fileToGet, prefix = "") { - return new GetObjectCommand({ Bucket: bucketName, Key: prefix + fileToGet }); - } - - getAWSPutCommand(bucketName, fileToGet, prefix = "") { - return new PutObjectCommand({ Bucket: bucketName, Key: prefix + fileToGet }); - } - - listAWSCommand(bucketName, prefix = "",) { - return new ListObjectsV2Command({ Bucket: bucketName, Prefix: prefix, Delimiter: "/",}); - } - - /** - * @description - Function to check whether file exists in specified bucket or not - * @param {string} bucketName - AWS bucket name - * @param {string} fileToGet - AWS File to check - * @param {string} prefix - `Optional` - Prefix for file path - * @param {function} cb - Callback function - */ - async fileExists(bucketName, fileToGet, prefix = "", cb) { - const params = { Bucket: bucketName, Key: prefix + fileToGet }; - const command = new HeadObjectCommand(params); - logger.info({ msg: "AWS__StorageService - fileExists called for bucketName " + bucketName + " for file " + params.Key }); - await this.client - .send(command) - .then((resp) => { - cb(null, resp); - }) - .catch((err) => { - cb(err); - }); - } - - /** - * @description - Provides a stream to read from a storage - * @param {string} bucketName - Bucket name or folder name in storage service - * @param {string} fileToGet - File path in storage service - */ - fileReadStream(_bucketName = undefined, fileToGet = undefined) { - return async (req, res, next) => { - let bucketName = _bucketName; - let fileToGet = req.params.slug.replace("__", "/") + "/" + req.params.filename; - logger.info({ msg: "AWS__StorageService - fileReadStream called for bucketName " + bucketName + " for file " + fileToGet }); - - if (fileToGet.includes(".json")) { - const streamToString = (stream) => - new Promise((resolve, reject) => { - const chunks = []; - stream.on("data", (chunk) => chunks.push(chunk)); - stream.on("error", (err) => { - reject(err); - }); - stream.on("end", () => { - resolve(Buffer.concat(chunks).toString("utf8")); - }); - }); - await this.client - .send(this.getAWSCommand(bucketName, fileToGet, undefined)) - .then((resp) => { - streamToString(_.get(resp, "Body")) - .then((data) => { - res.end(data); - }) - .catch((err) => { - storageLogger.s500(res, "AWS__StorageService : readStream error - Error 500", err, "Failed to execute readStream"); - }); - }) - .catch((error) => { - if (_.get(error, "$metadata.httpStatusCode") == 404) { - storageLogger.s404(res, "AWS__StorageService : readStream client send error - Error with status code 404", error, "File not found"); - } else { - storageLogger.s500(res, "AWS__StorageService : readStream client send error - Error 500", error, "Failed to display blob"); - } - }); - } else { - this.fileExists(bucketName, fileToGet, undefined, async (error, resp) => { - if (_.get(error, "$metadata.httpStatusCode") == 404) { - storageLogger.s404(res, "AWS__StorageService : fileExists error - Error with status code 404", error, "File does not exists"); - } else if (_.get(resp, "$metadata.httpStatusCode") == 200) { - const command = this.getAWSCommand(bucketName, fileToGet, undefined); - // `expiresIn` - The number of seconds before the presigned URL expires - const presignedURL = await getSignedUrl(this.client, command, { expiresIn: 3600 }); - const response = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: { - signedUrl: presignedURL, - }, - }; - res.status(200).send(this.apiResponse(response)); - } else { - storageLogger.s500(res, "AWS__StorageService : fileExists client send error - Error 500", "", "Failed to check file exists"); - } - }); - } - }; - } - - getFileProperties(_bucketName = undefined) { - return (req, res, next) => { - const bucketName = _bucketName; - const fileToGet = JSON.parse(req.query.fileNames); - logger.info({ msg: "AWS__StorageService - getFileProperties called for bucketName " + bucketName + " for file " + fileToGet }); - const responseData = {}; - if (Object.keys(fileToGet).length > 0) { - const getBlogRequest = []; - for (const [key, file] of Object.entries(fileToGet)) { - const req = { - bucketName: bucketName, - file: file, - reportname: key, - }; - getBlogRequest.push( - async.reflect((callback) => { - this.getBlobProperties(req, callback); - }) - ); - } - async.parallel(getBlogRequest, (err, results) => { - if (results) { - results.map((blob) => { - if (blob.error) { - responseData[_.get(blob, "error.reportname")] = blob.error; - } else { - responseData[_.get(blob, "value.reportname")] = { - lastModified: _.get(blob, "value.lastModified"), - reportname: _.get(blob, "value.reportname"), - statusCode: _.get(blob, "value.statusCode"), - fileSize: _.get(blob, "value.contentLength"), - }; - } - }); - const finalResponse = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: responseData, - }; - res.status(200).send(this.apiResponse(finalResponse)); - } - }); - } - }; - } - - async getBlobProperties(request, callback) { - this.fileExists(request.bucketName, request.file, undefined, (error, resp) => { - if (_.get(error, "$metadata.httpStatusCode") == 404) { - logger.error({ msg: "AWS__StorageService : getBlobProperties_fileExists error - Error with status code 404. File does not exists - " + request.file, error: error }); - callback({ msg: _.get(error, "name"), statusCode: _.get(error, "$metadata.httpStatusCode"), filename: request.file, reportname: request.reportname }); - } else if (_.get(resp, "$metadata.httpStatusCode") == 200) { - resp.reportname = request.reportname; - resp.statusCode = 200; - logger.info({ - msg: "AWS__StorageService : getBlobProperties_fileExists success with status code 200. File does exists - " + request.file, - statusCode: _.get(error, "$metadata.httpStatusCode"), - }); - callback(null, resp); - } else { - logger.error({ msg: "AWS__StorageService : getBlobProperties_fileExists client send error - Error 500 Failed to check file exists" }); - callback(true); - } - }); - } - - async getFileAsText(container = undefined, fileToGet = undefined, callback) { - const bucketName = container; - logger.info({ msg: "AWS__StorageService : getFileAsText called for bucket " + bucketName + " for file " + fileToGet }); - const streamToString = (stream) => - new Promise((resolve, reject) => { - const chunks = []; - stream.on("data", (chunk) => chunks.push(chunk)); - stream.on("error", (err) => { - reject(err); - }); - stream.on("end", () => { - resolve(Buffer.concat(chunks).toString("utf8")); - }); - }); - await this.client - .send(this.getAWSCommand(bucketName, fileToGet)) - .then((resp) => { - streamToString(_.get(resp, "Body")) - .then((data) => { - callback(null, data); - }) - .catch((err) => { - logger.error({ msg: "AWS__StorageService : getFileAsText error - Error 500", err: "Failed to execute getFileAsText" }); - callback(err); - }); - }) - .catch((error) => { - if (_.get(error, "$metadata.httpStatusCode") == 404) { - logger.error({ msg: "AWS__StorageService : getFileAsText client send error - Error with status code 404. File not found", error: error }); - } else { - logger.error({ msg: "AWS__StorageService : getFileAsText client send error - Error 500. Failed to display blob", error: error }); - } - callback(error); - }); - } - - blockStreamUpload(uploadContainer = undefined) { - return (req, res) => { - try { - const bucketName = uploadContainer; - const blobFolderName = new Date().toLocaleDateString(); - let form = new multiparty.Form(); - form.on("part", async (part) => { - if (part.filename) { - let size = part.byteCount - part.byteOffset; - let name = `${_.get(req, "query.deviceId")}_${Date.now()}.${_.get(part, "filename")}`; - logger.info({ - msg: "AWS__StorageService : blockStreamUpload Uploading file to bucket " + uploadContainer + " to folder " + blobFolderName + " for file name " + name + " with size " + size, - }); - let keyPath = uploadContainer + "/" + blobFolderName + "/" + name; - logger.info({ - msg: "AWS__StorageService : blockStreamUpload Uploading file to " + keyPath, - }); - try { - const parallelUploads3 = new Upload({ - client: this.client, - params: { Bucket: bucketName, Key: keyPath, Body: part }, - leavePartsOnError: false, - }); - parallelUploads3.on("httpUploadProgress", (progress) => { - let toStr; - for (let key in progress) { - if (progress.hasOwnProperty(key)) { - toStr += `${key}: ${progress[key]}` + ", "; - } - } - logger.info({ - msg: "AWS__StorageService : blockStreamUpload Uploading progress " + toStr, - }); - }); - await parallelUploads3 - .done() - .then((data) => { - const response = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: { - message: "Successfully uploaded to blob", - }, - }; - return res.status(200).send(this.apiResponse(response, "api.desktop.upload.crash.log")); - }) - .catch((err) => { - const response = { - responseCode: "SERVER_ERROR", - params: { - err: "SERVER_ERROR", - status: "failed", - errmsg: "Failed to upload to blob", - }, - result: {}, - }; - logger.error({ - msg: "AWS__StorageService : blockStreamUpload parallelUploads3 Failed to upload desktop crash logs to blob", - error: err, - }); - return res.status(500).send(this.apiResponse(response, "api.desktop.upload.crash.log")); - }); - } catch (e) { - const response = { - responseCode: "SERVER_ERROR", - params: { - err: "SERVER_ERROR", - status: "failed", - errmsg: "Failed to upload to blob", - }, - result: {}, - }; - logger.error({ - msg: "AWS__StorageService : blockStreamUpload try catch Failed to upload desktop crash logs to blob", - error: e, - }); - return res.status(500).send(this.apiResponse(response, "api.desktop.upload.crash.log")); - } - } - }); - form.parse(req); - } catch (error) { - const response = { - responseCode: "SERVER_ERROR", - params: { - err: "SERVER_ERROR", - status: "failed", - errmsg: "Failed to upload to blob", - }, - result: {}, - }; - logger.error({ - msg: "AWS__StorageService : blockStreamUpload Failed to upload desktop crash logs to blob", - error: error, - }); - return res.status(500).send(this.apiResponse(response, "api.desktop.upload.crash.log")); - } - }; - } - - apiResponse({ responseCode, result, params: { err, errmsg, status } }, id = "api.report") { - return { - id: id, - ver: "1.0", - ts: dateFormat(new Date(), "yyyy-mm-dd HH:MM:ss:lo"), - params: { - resmsgid: uuidv1(), - msgid: null, - status: status, - err: err, - errmsg: errmsg, - }, - responseCode: responseCode, - result: result, - }; - } - - async getPreSignedUrl(container, fileNames) { - const presignedURLs = await Promise.all( - fileNames.map(async (fileName) => { - const command = this.getAWSPutCommand(container, fileName, undefined); - const presignedURL = await getSignedUrl(this.client, command, { expiresIn: 3600 }) - return { fileName, presignedURL }; - }) - ) - return presignedURLs - } - - /** - * @description - Function to get file names S3 bucket for a specific date range - * @param {string} container - Bucket name to fetch the files from - * @param {string} container_prefix - Prefix of the path if the files are nested - * @param {string} type - Folder name/Type of data to fetch the files for - * @param {string} dateRange - Range of time interval, to get the files for - * @param {string} datasetId - Dataset Id to fetch the files for - */ - async filterDataByRange(container, container_prefix, type, dateRange, datasetId) { - let startDate = moment(dateRange.from); - let endDate = moment(dateRange.to); - let result = []; - let promises = []; - for (let analysisDate = startDate; analysisDate <= endDate; analysisDate = analysisDate.add(1, "days")) { - promises.push(new Promise((resolve, reject) => { - const pathPrefix = `${container_prefix}/${type}/${datasetId}/${analysisDate.format("YYYY-MM-DD")}`; - try { - resolve(this.client.send(this.listAWSCommand(container, pathPrefix,))); - } - catch (err) { console.log(err) } - })) - } - let S3Objects = await Promise.all(promises); - S3Objects.map((S3Object) => { - S3Object.Contents?.map((content) => { - result.push((content.Key || "")); - }) - }); - return (result); - } - - /** - * @description - Function to get file download URLs from S3 bucket - * @param {string} container - Bucket name to fetch the files from - * @param {Array} filesList - List of file keys obtained for generating signed urls for download - */ - async getFilesSignedUrls(container, filesList) { - const signedUrlsPromises = filesList.map((fileNameWithPrefix) => { - return new Promise(async (resolve, reject) => { - const command = this.getAWSCommand(container, fileNameWithPrefix); - const fileName = fileNameWithPrefix.split("/").pop(); - const presignedURL = await getSignedUrl(this.client, command, { expiresIn: globalConfig.exhaust_config.storage_url_expiry, }); - resolve({ [fileName]: presignedURL }); - }); - }); - const signedUrlsList = await Promise.all(signedUrlsPromises); - const periodWiseFiles = {}; - const files = []; - // Formatting response - signedUrlsList.map(async (fileObject) => { - const fileDetails = _.keys(fileObject); - const fileUrl = _.values(fileObject)[0]; - const period = getFileKey(fileDetails[0]); - if(_.has(periodWiseFiles, period)) - periodWiseFiles[period].push(fileUrl); - else { - periodWiseFiles[period] = []; - periodWiseFiles[period].push(fileUrl); - } - files.push(fileUrl); - }); - return { - expiresAt: moment().add(globalConfig.exhaust_config.storage_url_expiry, 'seconds').toISOString(), - files, - periodWiseFiles, - }; - } - - /** - * @description - Function to get file names S3 bucket for a specific date range - * @param {string} container - Bucket name to fetch the files from - * @param {string} container_prefix - Prefix of the path if the files are nested - * @param {string} type - Folder name/Type of data to fetch the files for - * @param {string} dateRange - Range of time interval, to get the files for - * @param {string} datasetId - Dataset Id to fetch the files for - */ - async getFiles(container, container_prefix, type, dateRange, datasetId) { - const filesList = await this.filterDataByRange(container, container_prefix, type, dateRange, datasetId); - const signedUrlsList = await this.getFilesSignedUrls(container, filesList); - return signedUrlsList; - } -} - -module.exports = AWSStorageService; diff --git a/api-service/src/v1/lib/client-cloud-services/AzureStorageService.js b/api-service/src/v1/lib/client-cloud-services/AzureStorageService.js deleted file mode 100644 index 4c2e053c..00000000 --- a/api-service/src/v1/lib/client-cloud-services/AzureStorageService.js +++ /dev/null @@ -1,405 +0,0 @@ -/** - * @file - Azure Storage Service - * @exports - `AzureStorageService` - * @since - 5.0.1 - * @version - 2.0.0 - * @implements - BaseStorageService - * - * @see {@link https://learn.microsoft.com/en-us/javascript/api/@azure/storage-blob/?view=azure-node-latest | Azure Blob Documentation} - * @see {@link https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/storage/storage-blob/MigrationGuide.md#uploading-a-blob-to-the-container | Azure Migration Guide} - */ - -const BaseStorageService = require("./BaseStorageService"); -const { logger } = require("@project-sunbird/logger"); -const _ = require("lodash"); -const { TextDecoder } = require("util"); -const { config: globalConfig } = require("../../configs/Config"); -const moment = require("moment"); -const { - BlobServiceClient, - StorageSharedKeyCredential, - generateBlobSASQueryParameters, - ContainerClient, -} = require("@azure/storage-blob"); -const { getFileKey } = require("../../utils/common"); -const READ = "r"; - -class AzureStorageService extends BaseStorageService { - constructor(config) { - super(); - if (!_.get(config, "identity") || !_.get(config, "credential")) { - throw new Error( - "Azure__StorageService :: Required configuration is missing" - ); - } - try { - this.sharedKeyCredential = new StorageSharedKeyCredential( - config?.identity, - config?.credential - ); - this.blobService = new BlobServiceClient( - `https://${config?.identity}.blob.core.windows.net`, - this.sharedKeyCredential - ); - this.containerClient = new ContainerClient( - `https://${config?.identity}.blob.core.windows.net/${globalConfig?.exhaust_config?.container}`, - this.sharedKeyCredential - ); - } catch (error) { - logger.info({ - msg: "Azure__StorageService - Unable to create Azure client", - }); - } - } - - async fileExists(container, fileToGet, callback) { - if (!container || !fileToGet || !callback) - throw new Error("Invalid arguments"); - logger.info({ - msg: - "Azure__StorageService - fileExists called for container " + - container + - " for file " + - fileToGet, - }); - const blobClient = this.blobService - .getContainerClient(container) - .getBlobClient(fileToGet); - try { - const blobProperties = await blobClient.getProperties(); - if (blobProperties) { - const response = { - exists: true, - }; - callback(null, response); - } - } catch (error) { - callback(error); - } - } - - /** - * @description - Retrieves a shared access signature token - * @param { string } container - Container name - * @param { string } blob - Blob to be fetched - * @param { azure.common.SharedAccessPolicy } sharedAccessPolicy - Shared access policy - * @param { azure.common.ContentSettingsHeaders } headers - Optional header values to set for a blob returned wth this SAS - * @return { string } - The shared access signature - */ - generateSharedAccessSignature( - container, - blob, - sharedAccessPolicy, - headers - ) { - const sasToken = generateBlobSASQueryParameters( - { - containerName: container, - blobName: blob, - ...sharedAccessPolicy.AccessPolicy, - ...headers, - }, - this.sharedKeyCredential - ).toString(); - return sasToken; - } - - /** - * @description - Retrieves a blob or container URL - * @param { string } container - Container name - * @param { string } blob - Blob to be fetched - * @param { string } SASToken - Shared Access Signature token - * @return { string } - Formatted URL string - */ - getUrl(container, blob, SASToken) { - const blobClient = this.blobService - .getContainerClient(container) - .getBlobClient(blob); - return `${blobClient.url}?${SASToken}`; - } - - async getBlobProperties(request, callback) { - logger.info({ - msg: - "Azure__StorageService - getBlobProperties called for container " + - request.container + - " for file " + - request.file, - }); - const blobClient = this.blobService - .getContainerClient(request.container) - .getBlobClient(request.file); - try { - const blobProperties = await blobClient.getProperties(); - if (blobProperties) { - blobProperties.reportname = request.reportname; - blobProperties.filename = request.file; - blobProperties.statusCode = 200; - callback(null, blobProperties); - } - } catch (error) { - logger.error({ - msg: "Azure__StorageService : readStream error - Error with status code 404", - }); - callback({ - msg: "NotFound", - statusCode: error.statusCode, - filename: request.file, - reportname: request.reportname, - }); - } - } - - async getFileAsText( - container = undefined, - fileToGet = undefined, - prefix = undefined, - callback - ) { - const blobClient = this.blobService - .getContainerClient(container) - .getBlobClient(fileToGet); - try { - const downloadResponse = await blobClient.download(0); - const textDecoder = new TextDecoder("utf-8"); - const content = []; - for await (const chunk of downloadResponse.readableStreamBody) { - content.push(textDecoder.decode(chunk)); - } - const text = content.join(""); - logger.info({ - msg: - "Azure__StorageService : getFileAsText success for container " + - container + - " for file " + - fileToGet, - }); - callback(null, text); - } catch (error) { - logger.error({ - msg: "Azure__StorageService : getFileAsText error => ", - error, - }); - delete error.request; - delete error.response; - delete error.details; - callback(error); - } - } - - upload(container, fileName, filePath, callback) { - throw new Error("AzureStorageService :: upload() must be implemented"); - } - - async getPreSignedUrl(container, fileName, prefix = undefined) { - if (prefix) { - fileName = prefix + fileName; - } - const presignedURL = await this.getSignedUrl( - container, - fileName, - globalConfig.exhaust_config.storage_url_expiry - ); - return presignedURL; - } - - /** - * @description - Generates a pre-signed URL for a specific operation on a file in Azure storage. - * @param {string} container - Azure container or bucket name. - * @param {string} filePath - Path to the file within the container. - * @param {number} expiresIn - Optional. Number of seconds before the pre-signed URL expires. - * @param {string} permission - Optional. The permission for the operation (e.g., READ, WRITE). - * @returns {Promise} - A promise that resolves to the pre-signed URL. - */ - getSignedUrl(container, filePath, expiresIn = 3600, permission = "") { - let startDate = new Date(); - let expiryDate = new Date(startDate); - expiryDate.setMinutes(startDate.getMinutes() + expiresIn); - startDate.setMinutes(startDate.getMinutes() - expiresIn); - let sharedAccessPolicy = { - AccessPolicy: { - permissions: permission !== "" ? permission : READ, - startsOn: startDate, - expiresOn: expiryDate, - }, - }; - let azureHeaders = {}; - let token = this.generateSharedAccessSignature( - container, - filePath, - sharedAccessPolicy, - azureHeaders - ); - let sasUrl = this.getUrl(container, filePath, token); - return Promise.resolve(sasUrl); - } - - /** - * @description - Generates a pre-signed URL for downloading a file from the Azure storage. - * @param {string} container - Azure container or bucket name. - * @param {string} filePath - Path to the file within the container. - * @param {number} expiresIn - Optional. Number of seconds before the pre-signed URL expires. - * @returns {Promise} - A promise that resolves to the downloadable URL. - */ - getDownloadableUrl(container, filePath, expiresIn = 3600) { - let startDate = new Date(); - let expiryDate = new Date(startDate); - expiryDate.setMinutes(startDate.getMinutes() + expiresIn); - let sharedAccessPolicy = { - AccessPolicy: { - permissions: READ, - startsOn: startDate, - expiresOn: expiryDate, - }, - }; - let azureHeaders = {}; - let token = this.generateSharedAccessSignature( - container, - filePath, - sharedAccessPolicy, - azureHeaders - ); - let downloadableUrl = this.getUrl(container, filePath, token); - return Promise.resolve(downloadableUrl); - } - - /** - * @description - Generates a ingestion specification for a file. - * @param {string} container - Bucket name. - * @param {string} filePath - Path to the file in the bucket. - * @returns {Promise} - A Promise that resolves to the Druid ingestion specification. - */ - getFileUrlForIngestion(container, filePath) { - let druidSpec = { - type: "azure", - uris: [`azure://${container}/${filePath}`], - }; - return Promise.resolve(druidSpec); - } - - /** - * @description - Function to get file download URLs from S3 bucket - * @param {string} container - Bucket name to fetch the files from - * @param {Array} filesList - List of file keys obtained for generating signed urls for download - */ - async getFilesSignedUrls(container, filesList) { - const signedUrlsPromises = filesList.map((fileNameWithPrefix) => { - return new Promise(async (resolve, reject) => { - const presignedURL = await this.getPreSignedUrl( - container, - fileNameWithPrefix - ); - const fileName = fileNameWithPrefix.split("/").pop(); - resolve({ [fileName]: presignedURL }); - }); - }); - const signedUrlsList = await Promise.all(signedUrlsPromises); - const periodWiseFiles = {}; - const files = []; - // Formatting response - signedUrlsList.map(async (fileObject) => { - const fileDetails = _.keys(fileObject); - const fileUrl = _.values(fileObject)[0]; - const period = getFileKey(fileDetails[0]); - if (_.has(periodWiseFiles, period)) - periodWiseFiles[period].push(fileUrl); - else { - periodWiseFiles[period] = []; - periodWiseFiles[period].push(fileUrl); - } - files.push(fileUrl); - }); - return { - expiresAt: moment() - .add(globalConfig.exhaust_config.storage_url_expiry, "seconds") - .toISOString(), - files, - periodWiseFiles, - }; - } - - /** - * @description - Function to get file names from container for a specific date range - * @param {string} container - container name to fetch the files from - * @param {string} container_prefix - Prefix of the path if the files are nested - * @param {string} type - Folder name/Type of data to fetch the files for - * @param {string} dateRange - Range of time interval, to get the files for - * @param {string} datasetId - Dataset Id to fetch the files for - */ - async filterDataByRange( - container, - container_prefix, - type, - dateRange, - datasetId - ) { - let startDate = moment(dateRange.from); - let endDate = moment(dateRange.to); - let result = []; - let promises = []; - for ( - let analysisDate = startDate; - analysisDate <= endDate; - analysisDate = analysisDate.add(1, "days") - ) { - promises.push( - new Promise(async (resolve, reject) => { - const pathPrefix = `${container_prefix}/${type}/${datasetId}/${analysisDate.format( - "YYYY-MM-DD" - )}`; - try { - const items = this.containerClient.listBlobsByHierarchy( - "/", - { prefix: pathPrefix } - ); - for await (const item of items) { - if (item && item.kind === "blob") resolve(item); - else resolve(null); - return; - } - } catch (err) { - console.log( - `Unable to list the blobs present in directory ${pathPrefix}` - ); - console.log(err); - reject(err); - return; - } - }) - ); - } - try { - result = await Promise.all(promises); - if(result.length > 0) result = result.map((item) => item.name); - return result; - } catch (err) { - console.log(err); - return []; - } - } - - /** - * @description - Function to get file names S3 bucket for a specific date range - * @param {String} container - Bucket name to fetch the files from - * @param {String} container_prefix - Prefix of the path if the files are nested - * @param {String} type - Folder name/Type of data to fetch the files for - * @param {String} dateRange - Range of time interval, to get the files for - * @param {String} datasetId - Dataset Id to fetch the files for - */ - async getFiles(container, container_prefix, type, dateRange, datasetId) { - const filesList = await this.filterDataByRange( - container, - container_prefix, - type, - dateRange, - datasetId - ); - const signedUrlsList = await this.getFilesSignedUrls( - container, - filesList - ); - return signedUrlsList; - } -} - -module.exports = AzureStorageService; diff --git a/api-service/src/v1/lib/client-cloud-services/BaseStorageService.js b/api-service/src/v1/lib/client-cloud-services/BaseStorageService.js deleted file mode 100644 index 0db90e37..00000000 --- a/api-service/src/v1/lib/client-cloud-services/BaseStorageService.js +++ /dev/null @@ -1,109 +0,0 @@ -/** - * @file - Base Cloud Storage Service - * @description - Provides the interface and base implementation of StorageService - * @module - BaseStorageService - * @exports - `BaseStorageService` - * @since - 5.0.1 - * @version - 1.0.0 - */ - -class BaseStorageService { - - /** - * @description - Download file from storage service - * @throws - Throws Exception if method is not invoked without declaration - * @param {String} container - Container name or folder name in storage service - * @param {String} fileToDownload - File path in storage service - * @param {Boolean} isDirectory - default `false` - */ - downloadFile(container, fileToDownload, isDirectory = false) { - throw new Error('BaseStorageService :: downloadFile() must be implemented'); - } - - /** - * @description - Retrieves a blob or container URL - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - getURI(container, filePath) { - throw new Error('BaseStorageService :: getURI() must be implemented'); - } - - /** - * @description - Provides a stream to read from a storage - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - fileReadStream(container, filePath) { - throw new Error('BaseStorageService :: fileReadStream() must be implemented'); - } - - /** - * @description - Checks whether or not a blob exists on the storage service - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - fileExists(container, filePath) { - throw new Error('BaseStorageService :: fileExists() must be implemented'); - } - - /** - * @description - Retrieves a shared access signature token or signed URL - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - getSharedAccessSignature(container, filePath) { - throw new Error('BaseStorageService :: getSharedAccessSignature() must be implemented'); - } - - /** - * @description - Get all user-defined metadata, standard HTTP properties, and system properties for the blob - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - */ - getFileProperties(container, filePath) { - throw new Error('BaseStorageService :: getFileProperties() must be implemented'); - } - - /** - * @description - Downloads a blob / file into a text string - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - * @param { function } callback - Callback function - */ - getFileAsText(container, filePath, callback) { - throw new Error('BaseStorageService :: getFileAsText() must be implemented'); - } - - /** - * @description - Stream upload file to storage service - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} filePath - File path in storage service - * @param { function } callback - Callback function - */ - blockStreamUpload(container, filePath, callback) { - throw new Error('BaseStorageService :: blockStreamUpload() must be implemented'); - } - - /** - * @description - Upload file to storage service - * @throws - Throws Exception if method is not invoked without declaration - * @param {string} container - Container name or folder name in storage service - * @param {string} fileName - File name of file to be uploaded - * @param {string} filePath - File path for file to be uploaded - * @param {object} options - Metadata options for file - * @param { function } callback - Callback function - */ - upload(container, fileName, filePath, callback) { - throw new Error('BaseStorageService :: upload() must be implemented'); - } -} - -module.exports = BaseStorageService; \ No newline at end of file diff --git a/api-service/src/v1/lib/client-cloud-services/GCPStorageService.js b/api-service/src/v1/lib/client-cloud-services/GCPStorageService.js deleted file mode 100644 index 4efb2245..00000000 --- a/api-service/src/v1/lib/client-cloud-services/GCPStorageService.js +++ /dev/null @@ -1,278 +0,0 @@ -/** - * @file - Google Cloud Provider (GCP) Storage Service - * @exports - `GCPStorageService` - * @since - 5.0.1 - * @version - 1.0.0 - * @implements - BaseStorageService - * @see {@link https://googleapis.dev/nodejs/storage/latest/Bucket.html | GCloud Bucket} - */ - -const BaseStorageService = require("./BaseStorageService"); -const storageLogger = require("./storageLogger"); -const { Storage } = require("@google-cloud/storage"); -const { logger } = require("@project-sunbird/logger"); -const async = require("async"); -const _ = require("lodash"); -const dateFormat = require("dateformat"); -const uuidv1 = require("uuid/v1"); - -class GCPStorageService extends BaseStorageService { - constructor(config) { - super(); - if (!_.get(config, "identity")) { - throw new Error("GCLOUD__StorageService :: Required configuration is missing - [identity]"); - } - if (!_.get(config, "credential")) { - throw new Error("GCLOUD__StorageService :: Required configuration is missing - [credential]"); - } - if (!_.get(config, "projectId")) { - throw new Error("GCLOUD__StorageService :: Required configuration is missing - [projectId]"); - } - this._storage = new Storage({ - credentials: { - client_email: _.get(config, "identity"), - private_key: _.get(config, "credential")?.toString(), - }, - projectId: _.get(config, "projectId"), - }); - } - - fileExists(bucketName, fileToGet, prefix = "", cb) { - const file = this._storage.bucket(bucketName).file(prefix + fileToGet); - logger.info({ msg: "GCLOUD__StorageService - fileExists called for bucketName " + bucketName + " for file " + prefix + fileToGet }); - file.exists((err, exists) => { - if (err) cb(err); - if (exists) { - cb(null, exists); - } else { - cb(null, null); - } - }); - } - - /** - * @description - Provides a stream to read from a storage - * @param {string} bucketName - Bucket name or folder name in storage service - * @param {string} fileToGet - File path in storage service - */ - fileReadStream(_bucketName = undefined, fileToGet = undefined) { - return async (req, res, next) => { - let bucketName = _bucketName; - let fileToGet = _bucketName + req.params.slug.replace("__", "/") + "/" + req.params.filename; - logger.info({ msg: "GCLOUD__StorageService - fileReadStream called for bucketName " + bucketName + " for file " + fileToGet }); - - if (fileToGet.includes(".json")) { - try { - const file = this._storage.bucket(bucketName).file(fileToGet); - const fileStream = file.createReadStream(); - const streamToString = (stream) => - new Promise((resolve, reject) => { - const chunks = []; - stream.on("data", (chunk) => chunks.push(chunk)); - stream.on("error", (err) => { - reject(err); - }); - stream.on("end", () => { - resolve(Buffer.concat(chunks).toString("utf8")); - }); - }); - streamToString(fileStream) - .then((data) => { - res.end(data); - }) - .catch((err) => { - if (_.get(err, "code") === 404) { - storageLogger.s404(res, "GCLOUD__StorageService : readStream error - Error " + _.get(err, "code") + " " + _.get(err, "message"), "", _.get(err, "message")); - } else { - storageLogger.s500(res, "GCLOUD__StorageService : readStream client send error - Error 500", err, "Failed to display blob"); - } - }); - } catch (error) { - storageLogger.s500(res, "GCLOUD__StorageService : readStream client send error - Error 500", error, "Failed to display blob"); - } - } else { - this.fileExists(bucketName, fileToGet, "", (error, fileExists) => { - if (error) { - storageLogger.s404(res, "GCLOUD__StorageService : readStream_fileExists error - Error 404", error, "File does not exists"); - } else if (fileExists) { - this.getSharedAccessSignature(bucketName, fileToGet, "", undefined, (err, presignedURL) => { - if (err) { - storageLogger.s500(res, "GCLOUD__StorageService : readStream_getSharedAccessSignature - Error 500. Failed to get shared access signature", err, err); - } else { - const response = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: { - signedUrl: presignedURL, - }, - }; - logger.info({ msg: "GCLOUD__StorageService - readStream_getSharedAccessSignature called for bucketName " + bucketName + " for file " + fileToGet }); - res.status(200).send(this.apiResponse(response)); - } - }); - } else { - storageLogger.s500(res, "GCLOUD__StorageService : readStream_fileExists error - Error 500. Failed to fetch or File does not exists", error, "Failed to fetch or File does not exists"); - } - }); - } - }; - } - - async getSharedAccessSignature(bucketName, fileToGet, prefix = "", expiresIn, cb) { - let expiryDate; - if (!expiresIn) { - let startDate = new Date(); - expiryDate = new Date(startDate); - expiryDate.setMinutes(startDate.getMinutes() + 3600); - startDate.setMinutes(startDate.getMinutes() - 3600); - } else { - expiryDate = expiresIn; - } - const _config = { action: "read", expires: expiryDate }; - const file = this._storage.bucket(bucketName).file(prefix + fileToGet); - await file - .getSignedUrl(_config) - .then((signedUrl) => { - cb(null, signedUrl && signedUrl.length > 0 && signedUrl[0]); - }) - .catch((err) => cb(_.get(err, "message"))); - } - - getFileProperties(_bucketName = undefined) { - return (req, res, next) => { - const bucketName = _bucketName; - const fileToGet = JSON.parse(req.query.fileNames); - logger.info({ msg: "GCLOUD__StorageService - getFileProperties called for bucketName " + bucketName + " for file " + fileToGet }); - const responseData = {}; - if (Object.keys(fileToGet).length > 0) { - const getBlogRequest = []; - for (const [key, file] of Object.entries(fileToGet)) { - const req = { - bucketName: bucketName, - file: file, - reportname: key, - }; - getBlogRequest.push( - async.reflect((callback) => { - this.getBlobProperties(req, callback); - }) - ); - } - async.parallel(getBlogRequest, (err, results) => { - if (results) { - results.map((blob) => { - if (blob.error) { - responseData[_.get(blob, "error.reportname")] = blob.error; - } else { - responseData[_.get(blob, "value.reportname")] = { - lastModified: _.get(blob, "value.updated"), - reportname: _.get(blob, "value.reportname"), - statusCode: _.get(blob, "value.statusCode"), - fileSize: _.get(blob, "value.size"), - }; - } - }); - const finalResponse = { - responseCode: "OK", - params: { - err: null, - status: "success", - errmsg: null, - }, - result: responseData, - }; - res.status(200).send(this.apiResponse(finalResponse)); - } - }); - } - }; - } - - async getBlobProperties(request, callback) { - const file = this._storage.bucket(request.bucketName).file(request.file); - file.getMetadata((err, metadata, resp) => { - if (err) { - logger.error({ msg: "GCLOUD__StorageService : getBlobProperties_getMetadata client send error - Error 500 Failed to check file exists", err: err }); - callback(err); - } else if (_.get(resp, "statusCode") == 404) { - logger.error({ msg: "GCLOUD__StorageService : getBlobProperties_getMetadata error - Error with status code 404. File does not exists - " + request.file, error: resp }); - callback({ msg: _.get(resp, "statusMessage"), statusCode: _.get(resp, "statusCode"), filename: request.file, reportname: request.reportname }); - } else if (_.get(resp, "statusCode") == 200) { - metadata.reportname = request.reportname; - metadata.statusCode = 200; - logger.info({ - msg: "GCLOUD__StorageService : getBlobProperties_getMetadata success with status code 200. File exists - " + request.file, - statusCode: _.get(resp, "statusCode"), - }); - callback(null, metadata); - } else { - logger.error({ msg: "GCLOUD__StorageService : getBlobProperties_getMetadata client send error - Error 500 Failed to check file exists" }); - callback(true); - } - }); - } - - async getFileAsText(container = undefined, fileToGet = undefined, callback) { - const bucketName = container; - logger.info({ msg: "GCLOUD__StorageService : getFileAsText called for bucket " + bucketName + " for file " + fileToGet }); - const file = this._storage.bucket(bucketName).file(container + fileToGet); - logger.info({ msg: "GCLOUD__StorageService : getFileAsText called for bucket " + bucketName + " for file " + container + fileToGet }); - const fileStream = file.createReadStream(); - const streamToString = (stream) => - new Promise((resolve, reject) => { - const chunks = []; - stream.on("data", (chunk) => chunks.push(chunk)); - stream.on("error", (err) => { - reject(err); - }); - stream.on("end", () => { - resolve(Buffer.concat(chunks).toString("utf8")); - }); - }); - streamToString(fileStream) - .then((data) => { - callback(null, data); - }) - .catch((err) => { - if (_.get(err, "code") === 404) { - callback(err); - logger.error({ msg: "GCLOUD__StorageService : getFileAsText error - Error " + _.get(err, "code") + " " + _.get(err, "message") }); - } else { - callback({ err: "Failed to display blob", statusCode: 500 }); - logger.error({ msg: "GCLOUD__StorageService : getFileAsText client send error - Error 500. Failed to display blob, Error ", err }); - } - }); - } - - blockStreamUpload(uploadContainer = undefined) { - return (req, res) => { - logger.info({ msg: "GCLOUD__StorageService : blockStreamUpload called for bucket" }); - return res.status(200); - }; - } - - apiResponse({ responseCode, result, params: { err, errmsg, status } }) { - return { - id: "api.report", - ver: "1.0", - ts: dateFormat(new Date(), "yyyy-mm-dd HH:MM:ss:lo"), - params: { - resmsgid: uuidv1(), - msgid: null, - status: status, - err: err, - errmsg: errmsg, - }, - responseCode: responseCode, - result: result, - }; - } - upload(container, fileName, filePath, callback) { - throw new Error("BaseStorageService :: upload() must be implemented"); - } -} -module.exports = GCPStorageService; diff --git a/api-service/src/v1/lib/client-cloud-services/index.js b/api-service/src/v1/lib/client-cloud-services/index.js deleted file mode 100644 index abf31642..00000000 --- a/api-service/src/v1/lib/client-cloud-services/index.js +++ /dev/null @@ -1,34 +0,0 @@ -/** - * @file - Entry file referencing Storage Service - * @description - Entry file referencing Storage Service - * @exports - `AzureStorageService`, `AWSStorageService` and 'GCPStorageService` - * @author - RAJESH KUMARAVEL - * @since - 5.0.3 - * @version - 1.0.0 - */ - -const AzureStorageService = require("./AzureStorageService"); -const AWSStorageService = require("./AWSStorageService"); -const GCPStorageService = require("./GCPStorageService"); - -/** - * Based on Environment Cloud Provider value - * Export respective Storage Service - */ - -function init(provider) { - switch (provider) { - case "azure": - return AzureStorageService; - break; - case "aws": - return AWSStorageService; - break; - case "gcloud": - return GCPStorageService; - break; - default: - throw new Error(`Client Cloud Service - ${provider} provider is not supported`); - } -} -module.exports = { init }; diff --git a/api-service/src/v1/lib/client-cloud-services/storageLogger.js b/api-service/src/v1/lib/client-cloud-services/storageLogger.js deleted file mode 100644 index 506036db..00000000 --- a/api-service/src/v1/lib/client-cloud-services/storageLogger.js +++ /dev/null @@ -1,56 +0,0 @@ -/** - * @file - Storage Service logger - * @exports - Different HTTP status log builder(s) - * @since - 5.0.1 - * @version - 1.0.0 - */ - -const { logger } = require('@project-sunbird/logger'); -const dateFormat = require('dateformat'); -const uuidv1 = require('uuid/v1'); - -module.exports = { - 's500': (res, logMessage = '', stack = '', errorMessage) => { - logger.error({ msg: logMessage, error: stack }); - const response = { - responseCode: "SERVER_ERROR", - params: { - err: "SERVER_ERROR", - status: "failed", - errmsg: errorMessage - }, - result: {} - } - res.status(500).send(apiResponse(response)); - }, - 's404': (res, logMessage = '', stack = '', errorMessage) => { - logger.error({ msg: logMessage, error: stack }); - const response = { - responseCode: "CLIENT_ERROR", - params: { - err: "CLIENT_ERROR", - status: "failed", - errmsg: errorMessage - }, - result: {} - } - res.status(404).send(apiResponse(response)); - } -}; - -function apiResponse({ responseCode, result, params: { err, errmsg, status } }) { - return { - 'id': 'api.report', - 'ver': '1.0', - 'ts': dateFormat(new Date(), 'yyyy-mm-dd HH:MM:ss:lo'), - 'params': { - 'resmsgid': uuidv1(), - 'msgid': null, - 'status': status, - 'err': err, - 'errmsg': errmsg - }, - 'responseCode': responseCode, - 'result': result - } -}; diff --git a/api-service/src/v1/lib/dispatcher/dispatcher.js b/api-service/src/v1/lib/dispatcher/dispatcher.js deleted file mode 100644 index a2f2eb4d..00000000 --- a/api-service/src/v1/lib/dispatcher/dispatcher.js +++ /dev/null @@ -1,54 +0,0 @@ -const winston = require("winston"); -const { KafkaDispatcher } = require("./kafka-dispatcher"); -require("winston-daily-rotate-file"); -require("./kafka-dispatcher"); -const appConfig = require("../../configs/Config") -const defaultFileOptions = { - filename: "dispatcher-%DATE%.log", - datePattern: "YYYY-MM-DD", - maxsize: appConfig.config.telemetry_service_config.maxsize, - maxFiles: "100", - zippedArchive: true, - json: true, -}; - - -class Dispatcher { - constructor(options) { - if (!options) throw new Error("Dispatcher options are required"); - this.logger = new winston.Logger({ level: "info" }); - this.options = options; - this.kafkaDispatcher = new KafkaDispatcher(this.options) - if (this.options.dispatcher == "kafka") { - this.logger.add(winston.transports.Kafka, this.options); - console.log("Kafka transport enabled !!!"); - } else if (this.options.dispatcher == "file") { - const config = Object.assign(defaultFileOptions, this.options); - this.logger.add(winston.transports.DailyRotateFile, config); - console.log("File transport enabled !!!"); - } else { - // Log to console - this.options.dispatcher = "console"; - const config = Object.assign({ json: true, stringify: (obj) => JSON.stringify(obj) }, this.options); - this.logger.add(winston.transports.Console, config); - console.log("Console transport enabled !!!"); - } - } - - dispatch(mid, message, params, callback) { - // this.logger.log("info", message, params, callback) - this.kafkaDispatcher.log("info", message, params, callback); - } - - health(callback) { - if (this.options.dispatcher === "kafka") { - this.logger.transports["kafka"].health(callback); - } else if (this.options.dispatcher === "console") { - callback(null, true); - } else { - callback("telemetry service is not healthy"); - } - } -} - -module.exports = { Dispatcher }; diff --git a/api-service/src/v1/lib/dispatcher/kafka-dispatcher.js b/api-service/src/v1/lib/dispatcher/kafka-dispatcher.js deleted file mode 100644 index 2da3e953..00000000 --- a/api-service/src/v1/lib/dispatcher/kafka-dispatcher.js +++ /dev/null @@ -1,60 +0,0 @@ -const winston = require("winston"); -const { Kafka } = require("kafkajs"); -const _ = require("lodash"); - -class KafkaDispatcher extends winston.Transport { - constructor(options) { - super(); - this.name = "kafka"; - this.options = options.kafka; - if (this.options.compression_type == "snappy") { - this.compression_attribute = 2; - } else if (this.options.compression_type == "gzip") { - this.compression_attribute = 1; - } else { - this.compression_attribute = 0; - } - this.client = new Kafka(this.options.config); - this.producer = this.client.producer(); - this.init(); - } - async init() { - await this.producer - .connect() - .then(() => { - console.log("kafka dispatcher is ready"); - }) - .catch((err) => { - console.error("Unable to connect to kafka", err.message); - }); - } - async log(level, msg, kafkaTopics, callback) { - try { - console.log(`pushing events to topic '${kafkaTopics[0]}'...`); - await this.producer.send({ - topic: kafkaTopics[0], - messages: [{ value: msg }], - }); - callback(null, true); - } catch (err) { - console.log(err); - callback(err); - } - } - - async health(callback) { - try { - const admin = this.client.admin(); - await admin.connect(); - await admin.listTopics(); - await admin.disconnect(); - callback(null, true); - } catch (error) { - callback(error); - } - } -} - -winston.transports.Kafka = KafkaDispatcher; - -module.exports = { KafkaDispatcher }; diff --git a/api-service/src/v1/lib/envVariables.js b/api-service/src/v1/lib/envVariables.js deleted file mode 100644 index b7bd5c6b..00000000 --- a/api-service/src/v1/lib/envVariables.js +++ /dev/null @@ -1,15 +0,0 @@ -const envVariables = { - level: process.env.telemetry_log_level || 'info', - localStorageEnabled: process.env.telemetry_local_storage_enabled || 'true', - dispatcher: process.env.telemetry_local_storage_type || 'kafka', - telemetryProxyEnabled: process.env.telemetry_proxy_enabled, - proxyURL: process.env.telemetry_proxy_url, - proxyAuthKey: process.env.telemetry_proxy_auth_key, - kafkaHost: `${process.env.kafka_host || 'localhost'}:${process.env.kafka_port || 9092}`, - topic: `${process.env.system_env || 'local'}.ingest`, - compression_type: process.env.telemetry_kafka_compression || 'none', - filename: process.env.telemetry_file_filename || 'telemetry-%DATE%.log', - maxsize: process.env.telemetry_file_maxsize || 10485760, - maxFiles: process.env.telemetry_file_maxfiles || '100', -} -module.exports = envVariables; \ No newline at end of file diff --git a/api-service/src/v1/lib/services/TelemetryService.js b/api-service/src/v1/lib/services/TelemetryService.js deleted file mode 100644 index a25679c6..00000000 --- a/api-service/src/v1/lib/services/TelemetryService.js +++ /dev/null @@ -1,124 +0,0 @@ -const uuidv1 = require("uuid/v1"), - // request = require("request"), - DispatcherClass = require("../dispatcher/dispatcher").Dispatcher; -const axios = require('axios').default; -const config = require("../../configs/Config").config; -const responseHandler = require("../../helpers/ResponseHandler").ResponseHandler; - -// TODO: Make this efficient. Implementation to be similar to typesafe config. Right now one configuration holds -// together all supported transport configurations - -class TelemetryService { - constructor(Dispatcher, config) { - this.config = config; - this.dispatcher = this.config.localStorageEnabled === "true" ? new Dispatcher(config) : undefined; - } - dispatch(req, res, ...params) { - const message = req.body; - message.did = req.get("x-device-id"); - message.channel = req.get("x-channel-id"); - message.pid = req.get("x-app-id"); - if (!message.mid) message.mid = uuidv1(); - message.syncts = new Date().getTime(); - - // add obsrv meta - const source = {meta: {id: "", connector_type: "api", version: config.version, entry_source: "api"}, trace_id: uuidv1()}; - const obsrvMeta = {syncts: new Date().getTime(), processingStartTime: new Date().getTime(), flags: {}, timespans: {}, error: {}, source: source}; - message.obsrv_meta = obsrvMeta; - - const data = JSON.stringify(message); - if (this.config.localStorageEnabled === "true" || this.config.telemetryProxyEnabled === "true") { - if (this.config.localStorageEnabled === "true" && this.config.telemetryProxyEnabled !== "true") { - // Store locally and respond back with proper status code - // this.dispatcher.dispatch(message.mid, data, this.getRequestCallBack(req, res)); - return new Promise(async (resolve, reject) => { - await this.dispatcher.dispatch(message.mid, data, params, (err, response) => { - if (err) { - return reject(err); - } else { - return resolve(response); - } - }); - }); - } else if (this.config.localStorageEnabled === "true" && this.config.telemetryProxyEnabled === "true") { - // Store locally and proxy to the specified URL. If the proxy fails ignore the error as the local storage is successful. Do a sync later - const options = this.getProxyRequestObj(req, data); - // request.post(options, (err, data) => { - // if (err) console.error("Proxy failed:", err); - // else console.log("Proxy successful! Server responded with:", data.body); - // }); - // this.dispatcher.dispatch(message.mid, data, this.getRequestCallBack(req, res)); - axios.post(options.url, options.body, { - headers: options.headers, - }).then((response) => { - console.log("Proxy successful! Server responded with:", data.body); - this.dispatcher.dispatch(message.mid, data, () => {}); - this.sendSuccess(req, res, { message: "the data has been successfully ingested" }); - }).catch((err) => { - console.error("Proxy failed:", err); - this.dispatcher.dispatch(message.mid, data, () => {}); - this.sendError(req, res, { message: err.message }); - }); - } else if (this.config.localStorageEnabled !== "true" && this.config.telemetryProxyEnabled === "true") { - // Just proxy - const options = this.getProxyRequestObj(req, data); - axios.post(options.url, options.body, { - headers: options.headers, - }).then( - this.sendSuccess(req, res, { message: "the data has been successfully ingested" }) - ).catch((err) => - this.sendError(req, res, { message: err.message }) - ) - // request.post(options, this.getRequestCallBack(req, res)); - } - } else { - this.sendError(req, res, { message: "Configuration error" }); - } - } - health(req, res) { - if (this.config.localStorageEnabled === "true") { - return new Promise(async (resolve, reject) => { - await this.dispatcher.health((err, response) => { - if (err) { - return reject(err); - } else { - return resolve(response); - } - }); - }); - } else if (this.config.telemetryProxyEnabled === "true") { - this.sendSuccess(req, res, { message: "Telemetry Proxy enabled" }); - } else { - this.sendError(req, res, { message: "Configuration error" }); - } - } - getRequestCallBack(req, res) { - let counter = 0; - return (err, data) => { - counter++; - if (counter > 1) return; - if (err) { - this.sendError(req, res, { message: err.message }); - } else { - this.sendSuccess(req, res, { message: "the data has been successfully ingested" }); - } - }; - } - sendError(req, res, options) { - responseHandler.errorResponse({ statusCode: 500, message: options.message, errCode: ["INTERNAL_SERVER_ERROR"] }, req, res); - } - sendSuccess(req, res, options) { - responseHandler.successResponse(req, res, { message: options.message }); - } - getProxyRequestObj(req, data) { - const headers = { authorization: "Bearer " + config.proxyAuthKey }; - if (req.get("content-type")) headers["content-type"] = req.get("content-type"); - if (req.get("content-encoding")) headers["content-encoding"] = req.get("content-encoding"); - return { - url: this.config.proxyURL, - headers: headers, - body: data, - }; - } -} -module.exports = new TelemetryService(DispatcherClass, config.telemetry_service_config); diff --git a/api-service/src/v1/managers/Extensions.ts b/api-service/src/v1/managers/Extensions.ts deleted file mode 100644 index 7e97f8c1..00000000 --- a/api-service/src/v1/managers/Extensions.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Application } from "express"; -import { extensions } from "../configs/Extensions"; -import path from "path"; - -export const loadExtensions = async (app: Application) => { - await loadRoutes(app) -} - -const loadRoutes = async (app: Application) => { - for (const extension of extensions) { - try { - const extensionPath = path.join(__dirname, "..", "..", "..", "node_modules", extension.id, extension.routePath) - let ExtensionRoute: any = await import(extensionPath) - const extensionRoute: IRouter = new ExtensionRoute.Router() - extensionRoute.init(app) - } catch (error) { - console.error(`Error while loading the extension of id ${extension.id} with path ${extension.routePath}`, error) - } - } -} - -export interface IRouter { - init(app: Application): void; - } - -export interface IRouterConstructor { - new(): IRouter; - } \ No newline at end of file diff --git a/api-service/src/v1/models/ConfigModels.ts b/api-service/src/v1/models/ConfigModels.ts deleted file mode 100644 index 538685aa..00000000 --- a/api-service/src/v1/models/ConfigModels.ts +++ /dev/null @@ -1,44 +0,0 @@ -export interface ExtractionConfig { - is_batch_event: boolean; - extraction_key: string; - dedup_config: DedupConfig; -} - -export interface DedupConfig { - drop_duplicates: boolean; - dedup_key: string; - dedup_period: number; -} - -export interface ValidationConfig { - validate: boolean; - mode: string; -} - -export interface DenormConfig { - redis_db_host: string; - redis_db_port: string; - denorm_fields: DenormFieldConfig; -} - -export interface DenormFieldConfig { - denorm_key: string; - redis_db: number; - denorm_out_field: string; -} - -export interface RouterConfig { - topic: string; -} - -export interface DatasetConfig { - data_key: string; - timestamp_key: string; - exclude_fields: string[]; - entry_topic: string; - redis_db_host: string; - redis_db_port: number; - redis_db: number; - index_data: boolean; -} - diff --git a/api-service/src/v1/models/DatasetModels.ts b/api-service/src/v1/models/DatasetModels.ts deleted file mode 100644 index c2e5dde1..00000000 --- a/api-service/src/v1/models/DatasetModels.ts +++ /dev/null @@ -1,57 +0,0 @@ -import { ValidationStatus } from "./ValidationModels"; -import { Request, Response } from "express"; - -export interface ISchemaGenerator { - generate: ((sample: Map) => any) | - ((sample: Map[]) => any); - process: ((sample: Map) => any) | - ((sample: Map[]) => any); -} -export interface IConnector { - connect(): any; - execute(sample: any, type?: any, topic?: any): any; - executeSql(sql: string[]): any; - close(): any -} - -// Interface with method for request body validation -export interface IValidator { - // Method to perform validation on the request body of a request - validate(data: any, id?: string): ValidationStatus | Promise; -} - -// Interface with method for request params validation -export interface QValidator extends IValidator { - // Method to perform validation on the query params of a request - validateQueryParams(data: any, id?: string): ValidationStatus | Promise; -} - -export interface Params { - status: string, - errmsg: string -} -export interface IResponse { - id: string, - ts: number, - ver: string, - params: Params, - responseCode: string, - result: any -} - -export interface Result { - data: object; - status: number; -} - -export enum DatasetStatus { - Live = 'Live', Retired = 'Retired', -} - -export enum TransformationMode { - Strict = 'Strict', Lenient = 'Lenient', -} - -export enum ValidationMode { - Strict = 'Strict', IgnoreNewFields = 'IgnoreNewFields', DiscardNewFields = 'DiscardNewFields', -} diff --git a/api-service/src/v1/models/IngestionModels.ts b/api-service/src/v1/models/IngestionModels.ts deleted file mode 100644 index 3b8b9498..00000000 --- a/api-service/src/v1/models/IngestionModels.ts +++ /dev/null @@ -1,35 +0,0 @@ -export interface IngestionSpecModel { - dimensions: any, - metrics: any, - flattenSpec: any -} - -export interface IOConfig { - topic: string, - bootstrapIp: string, - taskDuration: string, - completionTimeout: string -} -export interface TuningConfig { - maxRowPerSegment: number, - taskCount: number, -} - -export interface GranularitySpec { - segmentGranularity: string, - queryGranularity: string, - rollup: boolean -} - -export interface IngestionConfig { - dataset: string, - indexCol: string, - granularitySpec: GranularitySpec, - tuningConfig?: TuningConfig, - ioConfig?: IOConfig -} - -export interface IngestionSchemeRequest { - schema: Map[], - config: IngestionConfig -} diff --git a/api-service/src/v1/resources/Constants.json b/api-service/src/v1/resources/Constants.json deleted file mode 100644 index 5bb0f303..00000000 --- a/api-service/src/v1/resources/Constants.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "ERROR_MESSAGE": { - "INVALID_DATE_RANGE": "Invalid date range! make sure your range must be valid and cannot be more than ${allowedRange} days", - "NO_DATE_RANGE": "Invalid date range! the date range cannot be a null value", - "ROUTE_NOT_FOUND": "Route not found", - "INVALID_DATA_SOURCE": "limits for datasource does not exist!" - }, - "STATUS": { - "SUCCESS": "SUCCESS", - "FAILURE": "FAILED" - }, - "DATASET": { - "CREATED": "The data has been successfully ingested" - }, - "CONFIG": { - "DATASET_SAVED": "The dataset configuration has been saved successfully", - "DATASET_UPDATED": "The dataset configuration has been updated successfully", - "DATASOURCE_SAVED": "The datasource configuration has been saved successfully", - "DATASOURCE_UPDATED": "The datasource configuration has been updated successfully", - "DATASET_PUBLISHED": "The dataset configuration has been published successfully", - "DATASOURCE_PUBLISHED": "The datasource configuration has been published successfully", - "DATASET_RETIRED": "The dataset configuration has been retired successfully" - }, - "RECORD_NOT_FOUND": { - "message": "Record not found", - "status": 404, - "code": "NOT_FOUND" - }, - "DATASET_ID_NOT_FOUND": { - "message": "Dataset Id not found, Failed to ingest", - "status": 404, - "code": "NOT_FOUND" - }, - "DUPLICATE_RECORD": { - "message": "Record already Exists", - "status": 409, - "code": "CONFLICT" - }, - "FAILED_RECORD_UPDATE": { - "message": "Failed to save Record", - "status": 400, - "code": "BAD_REQUEST" - }, - "FAILED_RECORD_CREATE": { - "message": "Failed to save Record", - "status": 400, - "code": "BAD_REQUEST" - }, - "FAILED_RECORD_FETCH": { - "message": "Failed to fetch Records", - "status": 400, - "code": "BAD_REQUEST" - }, - "FAILED_SQL_QUERY": { - "message": "Failed to execute the request", - "status": 400, - "code": "BAD_REQUEST" - }, - "EMPTY_DATASET_ID": { - "message": "datasetId parameter in url cannot be empty", - "status": 400, - "code": "BAD_REQUEST" - }, - "INVALID_DATASET_CONFIG": { - "message": "Invalid dataset config provided for ingestion", - "status": 400, - "code": "BAD_REQUEST" - }, - "RECORD_SAVED": "The Record has been saved successfully", - "RECORD_UPDATED": "The Record has been update successfully", - "TABLE_NOT_FOUND": { - "message": "Table does not exist in database", - "status": 404, - "code": "NOT_FOUND" - }, - "EXHAUST": { - "NO_BACKUP_FILES": "Date range provided does not have any backup files" - }, - "INVALID_DATASOURCE_REF": { - "message": "Datasource ref must match ingestion spec datasource", - "status": 400, - "code": "BAD_REQUEST" - }, - "INVALID_TOPIC": { - "message": "topic name in ingestion spec must match dataset id", - "status": 400, - "code": "BAD_REQUEST" - }, - "DATASET_NOT_FOUND": { - "message": "Dataset id does not exist in database", - "status": 404, - "code": "NOT_FOUND" - }, - "INVALID_DATASOURCE": { - "message": "Datasource ${datasource} not available for querying", - "status": 404, - "code": "NOT_FOUND" - }, - "DUPLICATE_DENORM_FIELD": { - "message": "Duplicate found for denorm output key", - "status": 400, - "code": "BAD_REQUEST" - }, - "INGESTION_SUBMITTED": "ingestion spec has been submitted successfully", - "INGESTION_FAILED_ON_SAVE": "Failed to submit Ingestion Spec, record is not saved", - "FAILED_EXECUTING_QUERY": { - "status": 500, - "message": "Something went wrong while executing the query. Please try again later.", - "code": "INTERNAL_SERVER_ERROR" -} -} diff --git a/api-service/src/v1/resources/schemas/DataExhaustReq.json b/api-service/src/v1/resources/schemas/DataExhaustReq.json deleted file mode 100644 index 44b947cb..00000000 --- a/api-service/src/v1/resources/schemas/DataExhaustReq.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "oneOf": [ - { - "type": "object", - "properties": { - "type": { "type": "string" }, - "from": { "type": "string", "format": "date" }, - "to": { "type": "string", "format": "date" } - }, - "required": ["from", "to", "type"] - }, - { - "type": "object", - "properties": { - "type": { "type": "string" }, - "since": { "type": "string", "pattern": "^\\w{4}_\\d{1,2}_\\w{4}" } - }, - "required": ["since", "type"] - } - ] -} diff --git a/api-service/src/v1/resources/schemas/DataIngestionReq.json b/api-service/src/v1/resources/schemas/DataIngestionReq.json deleted file mode 100644 index 060c1b95..00000000 --- a/api-service/src/v1/resources/schemas/DataIngestionReq.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "type": "object", - "properties": { - "data": { - "type": "object" - } - }, - "required": [ - "data" - ] -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasetConfigDefault.ts b/api-service/src/v1/resources/schemas/DatasetConfigDefault.ts deleted file mode 100644 index f5ba0576..00000000 --- a/api-service/src/v1/resources/schemas/DatasetConfigDefault.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { config } from "../../configs/Config"; -import { DatasetStatus, ValidationMode } from "../../models/DatasetModels"; - -export const defaultConfig = { - "master": { - "dataset_config": { - "data_key": "", - "timestamp_key": "", - "exclude_fields": [], - "entry_topic": config.telemetry_service_config.kafka.topics.createMasterDataset, - "redis_db_host": config.redis_config.redis_host, - "redis_db_port": config.redis_config.redis_port, - "index_data": true, - "redis_db": 3 - }, - "validation_config": { - "validate": true, - "mode": ValidationMode.Strict, - }, - "extraction_config": { - "is_batch_event": false, - "extraction_key": "", - "dedup_config": { - "drop_duplicates": false, - "dedup_key": "", - "dedup_period": 604800, // 7 days - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - }, - "router_config": { - "topic": "" - }, - "tags": [], - "status": DatasetStatus.Live, - "created_by": "SYSTEM", - "updated_by": "SYSTEM" - }, - "dataset": { - "validation_config": { - "validate": true, - "mode": ValidationMode.Strict, - }, - "extraction_config": { - "is_batch_event": false, - "extraction_key": "", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 604800, // 7 days - }, - "router_config": { - "topic": "" - }, - "tags": [], - "dataset_config": { - "data_key": "", - "timestamp_key": "", - "exclude_fields": [], - "entry_topic": config.telemetry_service_config.kafka.topics.createDataset, - "redis_db_host": config.redis_config.redis_host, - "redis_db_port": config.redis_config.redis_port, - "index_data": true, - "redis_db": 0 // The default Redis database index. - }, - "status": DatasetStatus.Live, - "created_by": "SYSTEM", - "updated_by": "SYSTEM" - }, - "sourceConfig": { - "connector_type": '', - "connector_config": {}, - "status": DatasetStatus.Live, - "connector_stats": {}, - "created_by": 'SYSTEM', - "updated_by": 'SYSTEM' - } -} diff --git a/api-service/src/v1/resources/schemas/DatasetCreateReq.json b/api-service/src/v1/resources/schemas/DatasetCreateReq.json deleted file mode 100644 index fbabd4bc..00000000 --- a/api-service/src/v1/resources/schemas/DatasetCreateReq.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "type": { - "type": "string" - }, - "extraction_config": { - "type": "object" - }, - "validation_config": { - "type": "object" - }, - "dedup_config": { - "type": "object" - }, - "data_schema": { - "type": "object" - }, - "denorm_config": { - "type": "object" - }, - "router_config": { - "type": "object", - "properties": { - "topic": { - "type": "string" - } - }, - "required": ["topic"] - }, - "dataset_config": { - "type": "object", - "properties": { - "data_key": { - "type": "string" - }, - "timestamp_key": { - "type": "string" - }, - "exclude_fields": { - "type": "array", - "items": { - "type": "string" - } - }, - "entry_topic": { - "type": "string" - }, - "redis_db_host": { - "type": "string" - }, - "redis_db_port": { - "type": "number" - }, - "redis_db": { - "type": "number" - }, - "index_data": { - "type": "boolean" - } - } - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "type": "string", - "enum": ["Live", "Retired"] - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - }, - "created_date": { - "type": "string" - }, - "updated_date": { - "type": "string" - }, - "published_date": { - "type": "string" - } - }, - "required": ["dataset_id", "router_config", "type", "published_date"] -} diff --git a/api-service/src/v1/resources/schemas/DatasetListReq.json b/api-service/src/v1/resources/schemas/DatasetListReq.json deleted file mode 100644 index 3cfc29d6..00000000 --- a/api-service/src/v1/resources/schemas/DatasetListReq.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "type": "object", - "properties": { - "filters": { - "type": "object" - } - }, - "required": ["filters"] -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasetSourceConfigSaveReq.json b/api-service/src/v1/resources/schemas/DatasetSourceConfigSaveReq.json deleted file mode 100644 index ed789d87..00000000 --- a/api-service/src/v1/resources/schemas/DatasetSourceConfigSaveReq.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "connector_type": { - "type": "string" - }, - "connector_config": { - "type": "object" - }, - "status": { - "type": "string" - }, - "connector_stats": { - "type": "object" - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - } - }, - "required": [ - "connector_type", - "dataset_id" - ] -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasetSourceConfigUpdateReq.json b/api-service/src/v1/resources/schemas/DatasetSourceConfigUpdateReq.json deleted file mode 100644 index ed789d87..00000000 --- a/api-service/src/v1/resources/schemas/DatasetSourceConfigUpdateReq.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "connector_type": { - "type": "string" - }, - "connector_config": { - "type": "object" - }, - "status": { - "type": "string" - }, - "connector_stats": { - "type": "object" - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - } - }, - "required": [ - "connector_type", - "dataset_id" - ] -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasetUpdateReq.json b/api-service/src/v1/resources/schemas/DatasetUpdateReq.json deleted file mode 100644 index bd8fcbb4..00000000 --- a/api-service/src/v1/resources/schemas/DatasetUpdateReq.json +++ /dev/null @@ -1,99 +0,0 @@ -{ - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "name": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "type": { - "type": "string" - }, - "extraction_config": { - "type": "object" - }, - "validation_config": { - "type": "object" - }, - "dedup_config": { - "type": "object" - }, - "data_schema": { - "type": "object" - }, - "denorm_config": { - "type": "object" - }, - "router_config": { - "type": "object", - "properties": { - "topic": { - "type": "string" - } - }, - "required": ["topic"] - }, - "dataset_config": { - "type": "object", - "properties": { - "data_key": { - "type": "string" - }, - "timestamp_key": { - "type": "string" - }, - "exclude_fields": { - "type": "array", - "items": { - "type": "string" - } - }, - "entry_topic": { - "type": "string" - }, - "redis_db_host": { - "type": "string" - }, - "redis_db_port": { - "type": "number" - }, - "redis_db": { - "type": "number" - }, - "index_data": { - "type": "boolean" - } - } - }, - "tags": { - "type": "array", - "items": { - "type": "string" - } - }, - "status": { - "type": "string", - "enum": ["Live", "Retired"] - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - }, - "created_date": { - "type": "string" - }, - "updated_date": { - "type": "string" - }, - "published_date": { - "type": "string" - } - }, - "required": ["dataset_id"] -} diff --git a/api-service/src/v1/resources/schemas/DatasourceConfigDefault.json b/api-service/src/v1/resources/schemas/DatasourceConfigDefault.json deleted file mode 100644 index 48645bfd..00000000 --- a/api-service/src/v1/resources/schemas/DatasourceConfigDefault.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "retention_period": { - "enabled": "false" - }, - "archival_policy": { - "enabled": "false" - }, - "purge_policy": { - "enabled": "false" - }, - "backup_config": { - "enabled": "false" - }, - "status": "Live", - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "metadata": { - "aggregated": false, - "granularity": "day" - }, - "type": "druid" -} diff --git a/api-service/src/v1/resources/schemas/DatasourceSaveReq.json b/api-service/src/v1/resources/schemas/DatasourceSaveReq.json deleted file mode 100644 index 952bfaea..00000000 --- a/api-service/src/v1/resources/schemas/DatasourceSaveReq.json +++ /dev/null @@ -1,128 +0,0 @@ -{ - "type": "object", - "properties": { - "datasource": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "type": { - "type": "string", - "enum": ["druid", "datalake"] - }, - "datasource_ref": { - "type": "string" - }, - "retention_period": { - "type": "object" - }, - "archival_policy": { - "type": "object" - }, - "purge_policy": { - "type": "object" - }, - "backup_config": { - "type": "object" - }, - "status": { - "type": "string", - "enum": ["Live", "Retired"] - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - }, - "published_date": { - "type": "string" - }, - "metadata": { - "type": "object", - "properties": { - "aggregated": { - "type": "boolean" - }, - "granularity": { - "type": "string" - } - } - }, - "ingestion_spec": { - "type": "object" - } - }, - "required": ["dataset_id", "datasource", "published_date"], - - - "if": { - "properties": { - "type": { - "const": "druid" - } - } - }, - "then": { - "properties": { - "ingestion_spec": { - "type": "object" - } - } - }, - "else": { - "if": { - "properties": { - "type": { - "const": "datalake" - } - } - }, - "then": { - "properties": { - "ingestion_spec": { - "type": "object", - "properties": { - "dataset": { - "type": "string" - }, - "schema": { - "type": "object", - "properties": { - "partitionColumn": { - "type": "string" - }, - "primaryKey": { - "type": "string" - }, - "timestampColumn": { - "type": "string" - } - }, - "required": [ - "partitionColumn", - "primaryKey", - "timestampColumn" - ] - } - }, - "required": [ - "dataset", - "schema" - ] - } - } - }, - "else": { - "properties": { - "type": { - "enum": [ - "datalake", - "druid" - ] - } - } - } - } -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/DatasourceUpdateReq.json b/api-service/src/v1/resources/schemas/DatasourceUpdateReq.json deleted file mode 100644 index c60a8066..00000000 --- a/api-service/src/v1/resources/schemas/DatasourceUpdateReq.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "type": "object", - "properties": { - "datasource": { - "type": "string" - }, - "dataset_id": { - "type": "string" - }, - "type": { - "type": "string", - "enum": ["druid", "datalake"] - }, - "datasource_ref": { - "type": "string" - }, - "retention_period": { - "type": "object" - }, - "archival_policy": { - "type": "object" - }, - "purge_policy": { - "type": "object" - }, - "backup_config": { - "type": "object" - }, - "status": { - "type": "string", - "enum": ["Live", "Retired"] - }, - "created_by": { - "type": "string" - }, - "updated_by": { - "type": "string" - }, - "published_date": { - "type": "string" - }, - "metadata": { - "type": "object", - "properties": { - "aggregated": { - "type": "boolean" - }, - "granularity": { - "type": "string" - } - } - }, - "ingestion_spec": { - "type": "object" - } - }, - "required": ["dataset_id", "datasource"], - - "if": { - "properties": { - "type": { - "const": "druid" - } - } - }, - "then": { - "properties": { - "ingestion_spec": { - "type": "object" - } - } - }, - "else": { - "if": { - "properties": { - "type": { - "const": "datalake" - } - } - }, - "then": { - "properties": { - "ingestion_spec": { - "type": "object", - "properties": { - "dataset": { - "type": "string" - }, - "schema": { - "type": "object", - "properties": { - "table": { - "type": "string" - }, - "partitionColumn": { - "type": "string" - }, - "primaryKey": { - "type": "string" - }, - "timestampColumn": { - "type": "string" - } - }, - "required": [ - "partitionColumn", - "primaryKey", - "timestampColumn" - ] - } - }, - "required": [ - "dataset", - "schema" - ] - } - } - }, - "else": { - "properties": { - "type": { - "enum": [ - "datalake", - "druid" - ] - } - } - } - } -} diff --git a/api-service/src/v1/resources/schemas/QueryRequest.json b/api-service/src/v1/resources/schemas/QueryRequest.json deleted file mode 100644 index 9416c851..00000000 --- a/api-service/src/v1/resources/schemas/QueryRequest.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "type": "object", - "properties": { - "query": { - "type": "object", - "nullable": true - }, - "querySql": { - "type": "object", - "properties": { - "query": { - "type": "string" - } - }, - "required": [ - "query" - ], - "nullable": true - }, - "context": { - "type": "object", - "properties": { - "dataSource": { - "type": "string" - }, - "granularity": { - "type": "string", - "enum": ["five_minute", "ten_minute", "fifteen_minute", "thirty_minute", "hour", "six_hour", "eight_hour", "day", "week", "month", "quarter", "year"] - }, - "dataSourceType": { - "type": "string", - "enum": ["realtime", "datalake"] - } - } - } - }, - "oneOf": [ - { - "required": [ - "querySql" - ] - }, - { - "required": [ - "query" - ] - } - ] -} diff --git a/api-service/src/v1/resources/schemas/SchemaValidatorReq.json b/api-service/src/v1/resources/schemas/SchemaValidatorReq.json deleted file mode 100644 index 50dbe8af..00000000 --- a/api-service/src/v1/resources/schemas/SchemaValidatorReq.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "type": "object", - "properties": { - "filters": { - "type": "object", - "properties": { - "dataset_id": { - "type": "string", - "minLength": 3 - } - }, - "required": [ - "dataset_id" - ] - }, - "isLive": { - "type": "boolean" - }, - "event": { - "type": "object" - }, - "additionalProperties": false - }, - "required": [ - "filters", - "isLive", - "event" - ], - "additionalProperties": false -} \ No newline at end of file diff --git a/api-service/src/v1/resources/schemas/SubmitIngestionReq.json b/api-service/src/v1/resources/schemas/SubmitIngestionReq.json deleted file mode 100644 index 7c8020bc..00000000 --- a/api-service/src/v1/resources/schemas/SubmitIngestionReq.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "type": "object", - "properties": { - "type": { - "type": "string" - }, - "spec": { - "type": "object" - } - }, - "required": ["type", "spec"] -} diff --git a/api-service/src/v1/routes/Router.ts b/api-service/src/v1/routes/Router.ts deleted file mode 100644 index d716977a..00000000 --- a/api-service/src/v1/routes/Router.ts +++ /dev/null @@ -1,86 +0,0 @@ -import express from "express"; -import { config } from "../configs/Config"; -import { HTTPConnector } from "../connectors/HttpConnector"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { QueryService } from "../services/QueryService"; -import { ValidationService } from "../services/ValidationService"; -import { DatasetService } from "../services/DatasetService"; -import { KafkaConnector } from "../connectors/KafkaConnector"; -import { DataSourceService } from "../services/DataSourceService"; -import { DatasetSourceConfigService } from "../services/DatasetSourceConfigService"; -import { DbConnector } from "../connectors/DbConnector"; -import { routesConfig } from "../configs/RoutesConfig"; -import { IngestorService } from "../services/IngestorService"; -import { OperationType, telemetryAuditStart } from "../services/telemetry"; -import telemetryActions from "../data/telemetryActions"; -import { ClientCloudService } from "../services/ClientCloudService"; -import { WrapperService } from "../services/WrapperService"; -import { onRequest } from "../../v2/metrics/prometheus/helpers" -import { Entity } from "../../v2/types/MetricModel"; -import { HealthService } from "../services/HealthService"; -import { eventsValidationAgainstSchema } from "../services/EventsValidationAgainstSchema"; - -export const validationService = new ValidationService(); -const httpDruidConnector = new HTTPConnector(`${config.query_api.druid.host}:${config.query_api.druid.port}`) -export const queryService = new QueryService(httpDruidConnector); -export const kafkaConnector = new KafkaConnector() -export const dbConnector = new DbConnector(config.db_connector_config); -export const datasourceService = new DataSourceService(dbConnector, config.table_names.datasources); -export const datasetService = new DatasetService(dbConnector, config.table_names.datasets); -export const datasetSourceConfigService = new DatasetSourceConfigService(dbConnector, config.table_names.datasetSourceConfig); -export const ingestorService = new IngestorService(kafkaConnector,); -export const exhaustService = new ClientCloudService(config.exhaust_config.cloud_storage_provider, config.exhaust_config.cloud_storage_config); -export const wrapperService = new WrapperService(); -export const globalCache: any = new Map() -export const healthService = new HealthService(dbConnector, kafkaConnector, httpDruidConnector) -export const router = express.Router() -dbConnector.init() - -router.all("/datasets/v1/*", ResponseHandler.goneResponse) -router.all("/dataset/v1/*", ResponseHandler.goneResponse) -router.all("/datasources/v1/*", ResponseHandler.goneResponse) -router.all("/data/v1/*", ResponseHandler.goneResponse) -router.all("/template/v1/*", ResponseHandler.goneResponse) - -/** Query API(s) */ -router.post([`${routesConfig.query.native_query.path}`, `${routesConfig.query.native_query_with_params.path}`,], ResponseHandler.setApiId(routesConfig.query.native_query.api_id), telemetryAuditStart({ action: telemetryActions.nativeQuery, operationType: OperationType.GET }), onRequest({ entity: Entity.Data_out }), validationService.validateRequestBody, validationService.validateQuery, queryService.executeNativeQuery); -router.post([`${routesConfig.query.sql_query.path}`, `${routesConfig.query.sql_query_with_params.path}`,], ResponseHandler.setApiId(routesConfig.query.sql_query.api_id), telemetryAuditStart({ action: telemetryActions.sqlQuery, operationType: OperationType.GET }), onRequest({ entity: Entity.Data_out }), validationService.validateRequestBody, validationService.validateQuery, queryService.executeSqlQuery); - -/** Ingestor API */ -router.post(`${routesConfig.data_ingest.path}`, ResponseHandler.setApiId(routesConfig.data_ingest.api_id), telemetryAuditStart({ action: telemetryActions.ingestEvents, operationType: OperationType.CREATE }), onRequest({ entity: Entity.Data_in }), validationService.validateRequestBody, ingestorService.create); -router.post(`${routesConfig.tenant_ingest.path}`, ResponseHandler.setApiId(routesConfig.tenant_ingest.api_id), telemetryAuditStart({ action: telemetryActions.ingestEvents, operationType: OperationType.CREATE }), onRequest({ entity: Entity.Data_in }), validationService.validateRequestBody, ingestorService.tenant); - -/** Dataset APIs */ -router.post(`${routesConfig.config.dataset.save.path}`, ResponseHandler.setApiId(routesConfig.config.dataset.save.api_id), telemetryAuditStart({ action: telemetryActions.createDataset, operationType: OperationType.CREATE }), validationService.validateRequestBody, datasetService.save); -router.patch(`${routesConfig.config.dataset.update.path}`, ResponseHandler.setApiId(routesConfig.config.dataset.update.api_id), telemetryAuditStart({ action: telemetryActions.updateDataset, operationType: OperationType.UPDATE }), validationService.validateRequestBody, datasetService.update); -router.get(`${routesConfig.config.dataset.read.path}`, ResponseHandler.setApiId(routesConfig.config.dataset.read.api_id), telemetryAuditStart({ action: telemetryActions.readDataset, operationType: OperationType.GET }), datasetService.read); -router.post(`${routesConfig.config.dataset.list.path}`, ResponseHandler.setApiId(routesConfig.config.dataset.list.api_id), telemetryAuditStart({ action: telemetryActions.listDatasets, operationType: OperationType.LIST }), validationService.validateRequestBody, datasetService.list); - -/** Dataset Source Config APIs */ -router.post(`${routesConfig.config.dataset_source_config.save.path}`, ResponseHandler.setApiId(routesConfig.config.dataset_source_config.save.api_id), telemetryAuditStart({ action: telemetryActions.createDatasetSourceConfig, operationType: OperationType.CREATE }), validationService.validateRequestBody, datasetSourceConfigService.save); -router.patch(`${routesConfig.config.dataset_source_config.update.path}`, ResponseHandler.setApiId(routesConfig.config.dataset_source_config.update.api_id), telemetryAuditStart({ action: telemetryActions.updateDatasetSourceConfig, operationType: OperationType.UPDATE }), validationService.validateRequestBody, datasetSourceConfigService.update); -router.get(`${routesConfig.config.dataset_source_config.read.path}`, ResponseHandler.setApiId(routesConfig.config.dataset_source_config.read.api_id), telemetryAuditStart({ action: telemetryActions.getatasetSourceConfig, operationType: OperationType.GET }), datasetSourceConfigService.read); -router.post(`${routesConfig.config.dataset_source_config.list.path}`, ResponseHandler.setApiId(routesConfig.config.dataset_source_config.list.api_id), telemetryAuditStart({ action: telemetryActions.listDatasetSourceConfig, operationType: OperationType.LIST }), validationService.validateRequestBody, datasetSourceConfigService.list); - -/** DataSource API(s) */ -router.post(`${routesConfig.config.datasource.save.path}`, ResponseHandler.setApiId(routesConfig.config.datasource.save.api_id), telemetryAuditStart({ action: telemetryActions.createDatasource, operationType: OperationType.CREATE }), validationService.validateRequestBody, datasourceService.save); -router.patch(`${routesConfig.config.datasource.update.path}`, ResponseHandler.setApiId(routesConfig.config.datasource.update.api_id), telemetryAuditStart({ action: telemetryActions.updateDatasource, operationType: OperationType.UPDATE }), validationService.validateRequestBody, datasourceService.update); -router.get(`${routesConfig.config.datasource.read.path}`, ResponseHandler.setApiId(routesConfig.config.datasource.read.api_id), telemetryAuditStart({ action: telemetryActions.getDatasource, operationType: OperationType.GET }), datasourceService.read); -router.post(`${routesConfig.config.datasource.list.path}`, ResponseHandler.setApiId(routesConfig.config.datasource.list.api_id), telemetryAuditStart({ action: telemetryActions.listDatasource, operationType: OperationType.LIST }), validationService.validateRequestBody, datasourceService.list); - -/** Exhaust API(s) */ -router.get(`${routesConfig.exhaust.path}`, ResponseHandler.setApiId(routesConfig.exhaust.api_id), telemetryAuditStart({ action: telemetryActions.datasetExhaust, operationType: OperationType.GET }), validationService.validateRequestParams, exhaustService.getData); - -/*** Submit Ingestion API(s) */ -router.post(`${routesConfig.submit_ingestion.path}`, ResponseHandler.setApiId(routesConfig.submit_ingestion.api_id), telemetryAuditStart({ action: telemetryActions.submitIngestionSpec, operationType: OperationType.CREATE }), validationService.validateRequestBody, ingestorService.submitIngestion) - -/** Query Wrapper API(s) */ -router.post(routesConfig.query_wrapper.sql_wrapper.path, ResponseHandler.setApiId(routesConfig.query_wrapper.sql_wrapper.api_id), onRequest({ entity: Entity.Data_out }), wrapperService.forwardSql) -router.post(routesConfig.query_wrapper.native_post.path, ResponseHandler.setApiId(routesConfig.query_wrapper.native_post.api_id), onRequest({ entity: Entity.Data_out }), wrapperService.forwardNative) -router.get(routesConfig.query_wrapper.native_get.path, ResponseHandler.setApiId(routesConfig.query_wrapper.native_get.api_id), onRequest({ entity: Entity.Data_out }), wrapperService.forwardNativeGet) -router.delete(routesConfig.query_wrapper.native_delete.path, ResponseHandler.setApiId(routesConfig.query_wrapper.native_delete.api_id), onRequest({ entity: Entity.Data_out }), wrapperService.forwardNativeDel) -router.get(routesConfig.query_wrapper.druid_status.path, ResponseHandler.setApiId(routesConfig.query_wrapper.druid_status.api_id), wrapperService.nativeStatus) -router.get(routesConfig.health.path, ResponseHandler.setApiId(routesConfig.health.api_id), healthService.checkHealth.bind(healthService)) - -/* schema validator */ -router.post(`${routesConfig.schema_validator.path}`, ResponseHandler.setApiId(routesConfig.schema_validator.api_id), validationService.validateRequestBody, eventsValidationAgainstSchema) diff --git a/api-service/src/v1/services/ClientCloudService.ts b/api-service/src/v1/services/ClientCloudService.ts deleted file mode 100644 index 3b588462..00000000 --- a/api-service/src/v1/services/ClientCloudService.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { NextFunction, Request, Response } from 'express'; -import { ResponseHandler } from '../helpers/ResponseHandler'; -import httpStatus from 'http-status'; -import constants from '../resources/Constants.json'; -import { ingestorService } from '../routes/Router'; -import { getDateRange, isValidDateRange } from '../utils/common'; -import { config as globalConfig } from '../configs/Config'; -import CloudService from '../lib/client-cloud-services'; -import moment from "moment"; -import { DateRange } from '../models/ExhaustModels'; -import { updateTelemetryAuditEvent } from './telemetry'; - -export class ClientCloudService { - private cloudProvider: string - private client: any - private storage: any - private config: any - private momentFormat: string; - constructor(cloudProvider: string, config?: any) { - this.cloudProvider = cloudProvider - this.client = CloudService.init(this.cloudProvider) - this.config = config - this.storage = new this.client(this.config) - this.momentFormat = "YYYY-MM-DD"; - } - - verifyDatasetExists = async (datasetId: string) => { - const datasetRecord = await ingestorService.getDatasetConfig(datasetId); - return datasetRecord; - } - - getFromStorage = async (type: string | undefined, dateRange: DateRange, datasetId: string) => { - let resData: any = {}; - resData = await this.storage.getFiles( - globalConfig.exhaust_config.container, globalConfig.exhaust_config.container_prefix, type, dateRange, datasetId, - ) - return resData; - } - - getData = async (req: Request, res: Response, next: NextFunction) => { - const { params } = req; - const { datasetId } = params; - const { type } = req.query; - updateTelemetryAuditEvent({ request: req, object: { id: datasetId, type: "dataset", ver: "1.0.0" } }); - // Validations - if(type && globalConfig.exhaust_config.exclude_exhaust_types.includes(type.toString())) { - next({statusCode: 404, message: constants.RECORD_NOT_FOUND, errCode: httpStatus["404_NAME"],}) - return; - } - const datasetRecord = await this.verifyDatasetExists(datasetId); - if(!datasetRecord) { - next({statusCode: 404, message: constants.RECORD_NOT_FOUND, errCode: httpStatus["404_NAME"],}) - return; - } - const dateRange = getDateRange(req, res); - const isValidDates = isValidDateRange( - moment(dateRange.from, this.momentFormat), - moment(dateRange.to, this.momentFormat), - globalConfig.exhaust_config.maxQueryDateRange, - ); - if(!isValidDates) { - next({statusCode: 400, message: constants.ERROR_MESSAGE.INVALID_DATE_RANGE.replace("${allowedRange}", globalConfig.exhaust_config.maxQueryDateRange.toString()), errCode: httpStatus["400_NAME"],}) - return; - } - - // Fetch Data - const resData: any = await this.getFromStorage(type?.toString(), dateRange, datasetId); - if(resData.files.length === 0) { - next({statusCode: 404, message: constants.EXHAUST.NO_BACKUP_FILES, errCode: httpStatus["404_NAME"],}) - return; - } - ResponseHandler.successResponse(req, res, { status: 200, data: resData, }) - } -} diff --git a/api-service/src/v1/services/DataSourceService.ts b/api-service/src/v1/services/DataSourceService.ts deleted file mode 100644 index 1942535a..00000000 --- a/api-service/src/v1/services/DataSourceService.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import _ from 'lodash' -import { Datasources } from "../helpers/Datasources"; -import { findAndSetExistingRecord, updateTelemetryAuditEvent } from "./telemetry"; -import { DbUtil } from "../helpers/DbUtil"; -import constants from "../resources/Constants.json"; -import { ingestorService } from "../routes/Router"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { DatasetStatus, IConnector } from "../models/DatasetModels"; -import { config } from "../configs/Config"; - -const telemetryObject = { id: null, type: "datasource", ver: "1.0.0" }; - -export class DataSourceService { - private table: string - private dbConnector: IConnector; - private dbUtil: DbUtil - private errorHandler: ErrorResponseHandler; - - constructor(dbConnector: IConnector, table: string) { - this.dbConnector = dbConnector - this.table = table - this.dbUtil = new DbUtil(dbConnector, table) - this.errorHandler = new ErrorResponseHandler("DataSourceService"); - } - - public save = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasources = new Datasources(req.body) - const payload: any = datasources.setValues() - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(payload, 'id'), } }); - if(payload.type === config.datasource_storage_types.druid) await this.validateDruidDatasource(payload) - else this.validateLakehouseDatasource(payload) - await this.dbUtil.save(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - public update = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasources = new Datasources(req.body) - const payload: Record = datasources.setValues() - if(payload.type === config.datasource_storage_types.druid) await this.validateDruidDatasource(payload) - else this.validateLakehouseDatasource(payload) - await findAndSetExistingRecord({ dbConnector: this.dbConnector, table: this.table, request: req, filters: { "id": _.get(payload, 'id') }, object: { ...telemetryObject, id: _.get(payload, 'id') } }); - await this.dbUtil.upsert(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - public read = async (req: Request, res: Response, next: NextFunction) => { - try { - let status: any = req.query.status || DatasetStatus.Live - const id = req.params.datasourceId - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id } }); - await this.dbUtil.read(req, res, next, { id, status }) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - public list = async (req: Request, res: Response, next: NextFunction) => { - try { - const payload = req.body - await this.dbUtil.list(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - public validateDruidDatasource = async (payload: Record) => { - let datasetRecord = await ingestorService.getDatasetConfig(payload.dataset_id); - if (_.isEmpty(datasetRecord)) { - throw constants.DATASET_NOT_FOUND; - } - if (!_.isUndefined(payload.datasource_ref) && !_.isUndefined(payload.ingestion_spec?.spec?.dataSchema?.dataSource) && payload.datasource_ref !== payload.ingestion_spec?.spec?.dataSchema?.dataSource) { - throw constants.INVALID_DATASOURCE_REF; - } - if ( - !_.isUndefined(payload.dataset_id) && !_.isUndefined(payload.ingestion_spec) && datasetRecord.router_config.topic !== payload.ingestion_spec?.spec?.ioConfig?.topic) { - throw constants.INVALID_TOPIC; - } - } - - public validateLakehouseDatasource = (payload: Record) => { - if (_.isUndefined(payload.ingestion_spec.schema.table) || _.isEmpty(payload.ingestion_spec.schema.table)) { - payload.ingestion_spec.schema.table = payload.datasource_ref - } - payload.ingestion_spec.schema.table = payload.ingestion_spec.schema.table.replace(/-/g, '_') - } -} diff --git a/api-service/src/v1/services/DatasetService.ts b/api-service/src/v1/services/DatasetService.ts deleted file mode 100644 index 151db15d..00000000 --- a/api-service/src/v1/services/DatasetService.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import _ from 'lodash' -import { Datasets } from "../helpers/Datasets"; -import { findAndSetExistingRecord, updateTelemetryAuditEvent } from "./telemetry"; -import { DbUtil } from "../helpers/DbUtil"; -import { refreshDatasetConfigs } from "../helpers/DatasetConfigs"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { DatasetStatus, IConnector } from "../models/DatasetModels"; - -const telemetryObject = { id: null, type: "dataset", ver: "1.0.0" }; - -export class DatasetService { - private table: string - private dbConnector: IConnector; - private dbUtil: DbUtil; - private errorHandler: ErrorResponseHandler; - constructor(dbConnector: IConnector, table: string) { - this.dbConnector = dbConnector - this.table = table - this.dbUtil = new DbUtil(dbConnector, table) - this.errorHandler = new ErrorResponseHandler("DatasetService"); - } - - public save = async (req: Request, res: Response, next: NextFunction) => { - try { - const dataset = new Datasets(req.body) - const payload: any = dataset.setValues() - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(payload, 'dataset_id') } }); - await this.dbUtil.save(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - - public update = async (req: Request, res: Response, next: NextFunction) => { - try { - const dataset = new Datasets(req.body) - const payload = dataset.getValues() - await findAndSetExistingRecord({ dbConnector: this.dbConnector, table: this.table, request: req, filters: { "id": payload.id }, object: { ...telemetryObject, id: payload.id } }); - await this.dbUtil.update(req, res, next, payload) - await refreshDatasetConfigs() - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - - public read = async (req: Request, res: Response, next: NextFunction) => { - try { - let status: any = req.query.status || DatasetStatus.Live - const id = req.params.datasetId - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id } }); - await this.dbUtil.read(req, res, next, { id, status }) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - - public list = async (req: Request, res: Response, next: NextFunction) => { - try { - const payload = req.body - await this.dbUtil.list(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } -} diff --git a/api-service/src/v1/services/DatasetSourceConfigService.ts b/api-service/src/v1/services/DatasetSourceConfigService.ts deleted file mode 100644 index 0f19dd50..00000000 --- a/api-service/src/v1/services/DatasetSourceConfigService.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import _ from 'lodash' -import { DatasetSourceConfigs } from "../helpers/DatasetSourceConfigs"; -import { DbUtil } from "../helpers/DbUtil"; -import { findAndSetExistingRecord, updateTelemetryAuditEvent } from "./telemetry"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { DatasetStatus, IConnector } from "../models/DatasetModels"; - -const telemetryObject = { id: null, type: "dataset-source-config", ver: "1.0.0" }; - -export class DatasetSourceConfigService { - private table: string - private dbConnector: IConnector; - private dbUtil: DbUtil - private errorHandler: ErrorResponseHandler; - constructor(dbConnector: IConnector, table: string) { - this.dbConnector = dbConnector - this.table = table - this.dbUtil = new DbUtil(dbConnector, table) - this.errorHandler = new ErrorResponseHandler("DatasetSourceConfigService"); - } - - public save = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasetSourceConfig = new DatasetSourceConfigs(req.body) - const payload: any = datasetSourceConfig.setValues() - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(payload, 'id') } }); - await this.dbUtil.save(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - public update = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasetSourceConfig = new DatasetSourceConfigs(req.body) - const payload: Record = datasetSourceConfig.setValues() - await findAndSetExistingRecord({ dbConnector: this.dbConnector, table: this.table, request: req, filters: { "id": _.get(payload, 'id') }, object: { ...telemetryObject, id: _.get(payload, 'id') } }); - await this.dbUtil.upsert(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error) } - } - public read = async (req: Request, res: Response, next: NextFunction) => { - try { - let status: any = req.query.status || DatasetStatus.Live - const id = req.params.datasetId - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id } }); - await this.dbUtil.read(req, res, next, { id, status }) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - public list = async (req: Request, res: Response, next: NextFunction) => { - try { - const payload = req.body - await this.dbUtil.list(req, res, next, payload) - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } -} diff --git a/api-service/src/v1/services/EventsValidationAgainstSchema.ts b/api-service/src/v1/services/EventsValidationAgainstSchema.ts deleted file mode 100644 index b25e34f4..00000000 --- a/api-service/src/v1/services/EventsValidationAgainstSchema.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import * as _ from "lodash" -import { schemaValidation } from "../helpers/ValidationService"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { dbConnector } from "../routes/Router"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { config } from "../configs/Config"; - -export const eventsValidationAgainstSchema = async (req: Request, res: Response, next: NextFunction) => { - const errorHandler = new ErrorResponseHandler("DatasetService"); - try { - const isLive = _.get(req, "body.isLive"); - const event = _.get(req, "body.event"); - const filters = _.get(req, "body.filters") - let datasetRecord = await dbConnector.readRecords(isLive ? config.table_names.datasets : `${config.table_names.datasets}_draft`, { filters: filters }) - let schema = _.get(datasetRecord, "[0].data_schema") - - if (_.isEmpty(datasetRecord)) { - throw { - "message": `Dataset ${filters?.dataset_id} does not exists`, - "status": 404, - "code": "NOT_FOUND" - } - } - - const validateEventAgainstSchema = schemaValidation(event, _.omit(schema, "$schema")); - ResponseHandler.successResponse(req, res, { status: 200, data: { message: validateEventAgainstSchema?.message, isValid: validateEventAgainstSchema?.isValid } }); - } - catch (error) { - return errorHandler.handleError(req, res, next, error); - } -} \ No newline at end of file diff --git a/api-service/src/v1/services/HealthService.ts b/api-service/src/v1/services/HealthService.ts deleted file mode 100644 index 6a79ede9..00000000 --- a/api-service/src/v1/services/HealthService.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { AxiosInstance } from "axios"; -import { NextFunction, Request, Response } from "express"; -import _ from "lodash"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { IConnector } from "../models/DatasetModels"; -import { DbConnector } from "../connectors/DbConnector"; -import { KafkaConnector } from "../connectors/KafkaConnector"; - - - - -export class HealthService { - private dbConnector: DbConnector; - private kafkaConnector: KafkaConnector; - private httpDruidConnector: AxiosInstance; - private errorHandler: ErrorResponseHandler; - constructor(dbConnector: DbConnector, kafkaConnector: KafkaConnector, httpDruidConnector: IConnector) { - this.errorHandler = new ErrorResponseHandler("HealthService"); - this.httpDruidConnector = httpDruidConnector.connect() - this.dbConnector = dbConnector; - this.kafkaConnector = kafkaConnector; - } - - checkHealth(req: Request, res: Response, next: NextFunction) { - Promise.all([this.checkDruidHealth(), this.checkKafkaHealth(), this.checkPostgresHealth()]) - .then(() => { - ResponseHandler.successResponse(req, res, { status: 200, data: {} }) - }).catch(error => { - this.errorHandler.handleError(req, res, next, error) - }) - } - - private async checkDruidHealth() { - await this.httpDruidConnector.get("/status/health") - } - - private async checkKafkaHealth() { - await this.kafkaConnector.connect() - } - - private async checkPostgresHealth() { - await this.dbConnector.health() - } - -} diff --git a/api-service/src/v1/services/IngestorService.ts b/api-service/src/v1/services/IngestorService.ts deleted file mode 100644 index 482cd764..00000000 --- a/api-service/src/v1/services/IngestorService.ts +++ /dev/null @@ -1,105 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import constants from "../resources/Constants.json" -import { ResponseHandler } from "../helpers/ResponseHandler"; -import _ from 'lodash' -import { globalCache } from "../routes/Router"; -import { refreshDatasetConfigs } from "../helpers/DatasetConfigs"; -import { DatasetStatus, IConnector } from "../models/DatasetModels"; -import { wrapperService } from "../routes/Router"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -export class IngestorService { - private kafkaConnector: IConnector; - private errorHandler: ErrorResponseHandler; - constructor(kafkaConnector: IConnector,) { - this.kafkaConnector = kafkaConnector - this.errorHandler = new ErrorResponseHandler("IngestorService"); - this.init() - } - public init() { - this.kafkaConnector.connect() - .then(() => { - console.log("kafka connection arranged succesfully...") - }) - .catch((error: any) => { - console.log("error while connecting to kafka", error.message) - }) - } - - public create = async (req: Request, res: Response, next: NextFunction) => { - try { - const datasetId = this.getDatasetId(req); - const validData = await this.validateData(req.body.data, datasetId); - req.body = { ...req.body.data, dataset: datasetId }; - const topic = await this.getTopic(datasetId); - await this.kafkaConnector.execute(req, res, topic); - ResponseHandler.successResponse(req, res, { status: 200, data: { message: constants.DATASET.CREATED } }); - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - - public tenant = async (req: Request, res: Response, next: NextFunction) => { - try { - let datasetId = this.getDatasetId(req); - const tenantId = _.get(req.headers, 'x-tenant-id', "default"); - datasetId = `${tenantId}-${datasetId}`; - const validData = await this.validateData(req.body.data, datasetId); - req.body = { ...req.body.data, dataset: datasetId }; - const topic = await this.getTopic(datasetId); - await this.kafkaConnector.execute(req, res, topic); - ResponseHandler.successResponse(req, res, { status: 200, data: { message: constants.DATASET.CREATED } }); - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - - public submitIngestion = async (req: Request, res: Response, next: NextFunction) => { - try { - await wrapperService.submitIngestion(req.body) - ResponseHandler.successResponse(req, res, { status: 200, data: { message: constants.INGESTION_SUBMITTED } }); - } - catch (error: any) { this.errorHandler.handleError(req, res, next, error, false) } - } - private getDatasetId(req: Request) { - let datasetId = req.params.datasetId.trim() - if (!_.isEmpty(datasetId)) return datasetId - throw constants.EMPTY_DATASET_ID - } - - public async getDatasetConfig(datasetId: string) { - let datasetConfigList = globalCache.get("dataset-config"); - if (!datasetConfigList) await refreshDatasetConfigs(); - - datasetConfigList = globalCache.get("dataset-config"); - const datasetRecord = datasetConfigList.find((record: any) => record.id === datasetId && record.status === DatasetStatus.Live); - // Return record if present in cache - if (datasetRecord) return datasetRecord; - else { // Refresh dataset configs cache in case record present in cache - await refreshDatasetConfigs(); - const datasetConfigList = globalCache.get("dataset-config"); - const datasetRecord = datasetConfigList.find((record: any) => record.id === datasetId && record.status === DatasetStatus.Live); - return datasetRecord; - } - } - - private async getTopic(datasetId: string) { - const datasetRecord = await this.getDatasetConfig(datasetId); - if (!datasetRecord) throw constants.DATASET_ID_NOT_FOUND; - return datasetRecord.dataset_config.entry_topic; - } - - private async validateData(data: any, datasetId: string) { - const datasetRecord = await this.getDatasetConfig(datasetId); - if (!datasetRecord) throw constants.DATASET_ID_NOT_FOUND; - if(_.has(datasetRecord, "extraction_config") && _.get(datasetRecord, ["extraction_config", "is_batch_event"])) { - if( - _.has(data, _.get(datasetRecord, ["extraction_config", "extraction_key"])) && - _.has(data, _.get(datasetRecord, ["extraction_config", "batch_id"])) - ) - return data; - else if (_.has(data, "event")) - return data; - else throw constants.INVALID_DATASET_CONFIG; - } else { - if(_.has(data, "event")) - return data; - else throw constants.INVALID_DATASET_CONFIG; - } - } -} diff --git a/api-service/src/v1/services/QueryService.ts b/api-service/src/v1/services/QueryService.ts deleted file mode 100644 index f0b8706a..00000000 --- a/api-service/src/v1/services/QueryService.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { AxiosInstance } from "axios"; -import { NextFunction, Request, Response } from "express"; -import _ from "lodash"; -import { config } from "../configs/Config"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { IConnector } from "../models/DatasetModels"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { updateTelemetryAuditEvent } from "./telemetry"; -import { executeLakehouseQuery } from "../helpers/LakehouseUtil"; - -const telemetryObject = { id: null, type: "datasource", ver: "1.0.0" }; - -export class QueryService { - private connector: AxiosInstance; - private errorHandler: ErrorResponseHandler; - constructor(connector: IConnector) { - this.connector = connector.connect(); - this.errorHandler = new ErrorResponseHandler("QueryService"); - } - - public executeNativeQuery = async (req: Request, res: Response, next: NextFunction) => { - try { - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(req, 'body.context.dataSource')} }); - var result = await this.connector.post(config.query_api.druid.native_query_path, req.body.query); - var mergedResult = result.data; - if (req.body.query.queryType === "scan" && result.data) { - mergedResult = result.data.map((item: Record) => { - return item.events; - }); - } - ResponseHandler.successResponse(req, res, { status: result.status, data: _.flatten(mergedResult) }); - - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } - }; - - public executeSqlQuery = async (req: Request, res: Response, next: NextFunction) => { - try { - let result: any = {} - updateTelemetryAuditEvent({ request: req, object: { ...telemetryObject, id: _.get(req, 'body.context.dataSource')} }); - if (req.body?.context?.dataSourceType === config.query_api.lakehouse.queryType) { - result.data = await executeLakehouseQuery(req.body.querySql.query) - } - else{ - result = await this.connector.post(config.query_api.druid.sql_query_path, req.body.querySql); - } - ResponseHandler.successResponse(req, res, { status: result.status || 200, data: result.data }); - } catch (error: any) { this.errorHandler.handleError(req, res, next, error, false); } - } -} diff --git a/api-service/src/v1/services/ValidationService.ts b/api-service/src/v1/services/ValidationService.ts deleted file mode 100644 index 24865a19..00000000 --- a/api-service/src/v1/services/ValidationService.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { NextFunction, Request, Response } from "express"; -import httpStatus from "http-status"; -import { IValidator, QValidator } from "../models/DatasetModels"; -import { ValidationStatus } from "../models/ValidationModels"; -import { QueryValidator } from "../validators/QueryValidator"; -import { RequestsValidator } from "../validators/RequestsValidator"; - -export class ValidationService { - - private request: QValidator; - private query: IValidator; - constructor() { - this.request = new RequestsValidator() - this.query = new QueryValidator() - } - - public validateRequestBody = async (req: Request, res: Response, next: NextFunction) => { - const status: ValidationStatus = await this.request.validate(req.body, (req as any).id) - status.isValid ? - next() - : next({ statusCode: httpStatus.BAD_REQUEST, message: status.message || "", errCode: status.code }) - }; - - public validateRequestParams = async (req: Request, res: Response, next: NextFunction) => { - const status: ValidationStatus = await this.request.validateQueryParams(req.query, (req as any).id) - status.isValid ? - next() - : next({ statusCode: httpStatus.BAD_REQUEST, message: status.message || "", errCode: status.code }) - }; - - public validateQuery = async (req: Request, res: Response, next: NextFunction) => { - const status: ValidationStatus = await this.query.validate(req.body, (req as any).id) - status.isValid ? - next() - : next({ statusCode: httpStatus.BAD_REQUEST, message: status.message || "", errCode: status.code }) - }; -} diff --git a/api-service/src/v1/services/telemetry.ts b/api-service/src/v1/services/telemetry.ts deleted file mode 100644 index e63d1d09..00000000 --- a/api-service/src/v1/services/telemetry.ts +++ /dev/null @@ -1,187 +0,0 @@ -import { Request, Response, NextFunction } from "express" -import { v4 } from 'uuid'; -import _ from 'lodash'; -import { config as appConfig } from "../configs/Config"; -import { Kafka } from 'kafkajs'; - -const env = _.get(appConfig, 'env') -const telemetryTopic = _.get(appConfig, 'telemetry_dataset'); -const brokerServers = _.get(appConfig, 'telemetry_service_config.kafka.config.brokers'); - -export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } - -const kafka = new Kafka({ clientId: telemetryTopic, brokers: brokerServers }); -const telemetryEventsProducer = kafka.producer(); -telemetryEventsProducer.connect().catch(err => console.error("Unable to connect to kafka", err.message)); - -const getDefaults = () => { - return { - eid: 'AUDIT', - ets: Date.now(), - ver: '1.0.0', - mid: v4(), - actor: { - id: "SYSTEM", - type: 'User' - }, - context: { - env, - sid: v4(), - pdata: { - id: `${env}.api.service`, - ver: '1.0.0' - } - }, - object: {}, - edata: {} - }; -}; - -const getDefaultEdata = ({ action }: any) => ({ - startEts: Date.now(), - type: null, - object: {}, - fromState: "inprogress", - toState: "completed", - edata: { - action, - props: [], - transition: { - timeUnit: "ms", - duration: 0 - } - } -}) - -const sendTelemetryEvents = async (event: Record) => { - telemetryEventsProducer.send({ topic: telemetryTopic, messages: [{ value: JSON.stringify(event) }] }).catch(console.log) -} - -const transformProps = (body: Record) => { - return _.map(_.entries(body), (payload) => { - const [key, value] = payload; - return { - property: key, - ov: null, - nv: value - } - }) -} - -export const setAuditState = (state: string, req: any) => { - if (state && req) { - _.set(req.auditEvent, "toState", state); - } -} - -const setAuditEventType = (operationType: any, request: any) => { - switch (operationType) { - case OperationType.CREATE: { - _.set(request, 'auditEvent.type', 'create'); - break; - } - case OperationType.UPDATE: { - _.set(request, 'auditEvent.type', 'update'); - break; - } - case OperationType.PUBLISH: { - _.set(request, 'auditEvent.type', 'publish'); - break; - } - case OperationType.RETIRE: { - _.set(request, 'auditEvent.type', 'retire'); - break; - } - case OperationType.LIST: { - _.set(request, 'auditEvent.type', 'list'); - break; - } - case OperationType.GET: { - _.set(request, 'auditEvent.type', 'get'); - break; - } - default: - break; - } -} - -export const telemetryAuditStart = ({ operationType, action }: any) => { - return async (request: any, response: Response, next: NextFunction) => { - try { - const body = request.body || {}; - request.auditEvent = getDefaultEdata({ action }); - const props = transformProps(body); - _.set(request, 'auditEvent.edata.props', props); - setAuditEventType(operationType, request); - } catch (error) { - console.log(error); - } finally { - next(); - } - } -} - -export const processAuditEvents = (request: Request, response: Response) => { - const auditEvent: any = _.get(request, 'auditEvent'); - if (auditEvent) { - const { startEts, object = {}, edata = {}, toState, fromState }: any = auditEvent; - const endEts = Date.now(); - const duration = startEts ? (endEts - startEts) : 0; - _.set(auditEvent, 'edata.transition.duration', duration); - if (toState && fromState) { - _.set(auditEvent, 'edata.transition.toState', toState); - _.set(auditEvent, 'edata.transition.fromState', fromState); - } - const telemetryEvent = getDefaults(); - _.set(telemetryEvent, 'edata', edata); - _.set(telemetryEvent, 'object', { ...(object.id && object.type && { ...object, ver: '1.0.0' }) }); - sendTelemetryEvents(telemetryEvent); - } -} - -export const interceptAuditEvents = () => { - return (request: Request, response: Response, next: NextFunction) => { - response.on('finish', () => { - const statusCode = _.get(response, 'statusCode'); - const isError = statusCode && statusCode >= 400; - !isError && processAuditEvents(request, response); - }) - next(); - } -} - -export const updateTelemetryAuditEvent = ({ currentRecord, request, object = {} }: Record) => { - const auditEvent = request?.auditEvent; - _.set(request, 'auditEvent.object', object); - if (currentRecord) { - const props = _.get(auditEvent, 'edata.props'); - const updatedProps = _.map(props, (prop: Record) => { - const { property, nv } = prop; - const existingValue = _.get(currentRecord, property); - return { property, ov: existingValue, nv }; - }); - _.set(request, 'auditEvent.edata.props', updatedProps); - } -} - -export const findAndSetExistingRecord = async ({ dbConnector, table, filters, request, object = {} }: Record) => { - const auditEvent = request?.auditEvent; - if (dbConnector && table && filters && _.get(auditEvent, 'type') === "update") { - try { - _.set(request, 'auditEvent.object', object); - const records = await dbConnector.execute("read", { table, fields: { filters } }) - const existingRecord = _.first(records); - if (existingRecord) { - const props = _.get(auditEvent, 'edata.props'); - const updatedProps = _.map(props, (prop: Record) => { - const { property, nv } = prop; - const existingValue = _.get(existingRecord, property); - return { property, ov: existingValue, nv }; - }); - _.set(request, 'auditEvent.edata.props', updatedProps); - } - } catch (error) { - setAuditState("failed", request); - } - } -} diff --git a/api-service/src/v1/test/ClientCloudService.spec.ts b/api-service/src/v1/test/ClientCloudService.spec.ts deleted file mode 100644 index 398ce60f..00000000 --- a/api-service/src/v1/test/ClientCloudService.spec.ts +++ /dev/null @@ -1,128 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import httpStatus from "http-status"; -import { TestExhaust } from "./Fixtures"; -import { config } from "./Config"; -import constants from "../resources/Constants.json"; -import { dbConnector, globalCache, exhaustService } from "../routes/Router"; -import chaiSpies from 'chai-spies' -import { describe, it } from 'mocha'; -import { config as appConfig } from "../configs/Config"; -import moment from "moment"; -import { DatasetStatus } from "../models/DatasetModels"; -chai.use(chaiSpies) -chai.should(); -chai.use(chaiHttp); - -describe("AWS Cloud Storage", () => { - beforeEach(() => { - chai.spy.on(globalCache, "get", () => { - return [{ "id": ":datasetId", "dataset_id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" } }] - }) - }) - - afterEach(() => { - chai.spy.restore() - }) - - it("it should return 404 if dataset record not found", (done) => { - chai.spy.restore() - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([{}]) - }) - chai.spy.on(globalCache, "get", () => { - return [] - }) - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.VALID_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - res.body.result.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]) - done() - }) - }); - - it("it should raise error when from or to have invalid dates", (done) => { - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.INVALID_DATE_RANGE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - res.body.result.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]) - done() - }) - }); - - it("it should raise error when time interval is greater than 31 days", (done) => { - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.DATE_RANGE_OVER_LIMIT) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - res.body.params.errmsg.should.be.eq(constants.ERROR_MESSAGE.INVALID_DATE_RANGE.replace("${allowedRange}", `${config.exhaustMaxDateRange}`)) - res.body.result.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]) - done() - }) - }); - - it("it should return 404 when no files exist for given date range", (done) => { - chai.spy.on(exhaustService, "getFromStorage", () => { - return Promise.resolve({ - expiresAt: moment().add(appConfig.exhaust_config.storage_url_expiry, "seconds").toISOString(), - files: [], - periodWiseFiles: {}, - }); - }) - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.VALID_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - res.body.result.should.be.a("object") - res.body.params.errmsg.should.be.eq(constants.EXHAUST.NO_BACKUP_FILES) - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]) - done() - }) - }); - - it("it should return data for a valid request", (done) => { - const timestamp = moment().add(config.storage_url_expiry, "seconds").toISOString() - const periodWiseData = { - "2023-06-15": ["file1", "file2"], - }; - chai.spy.on(exhaustService, "getFromStorage", () => { - return Promise.resolve({ - expiresAt: timestamp, - files: ["file1", "file2"], - periodWiseFiles: periodWiseData, - }); - }) - chai - .request(app) - .get(config.apiExhaustEndPoint) - .query(TestExhaust.VALID_REQUEST) - .end((err, res) => { - res.should.have.status(httpStatus.OK) - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - res.body.result.expiresAt.should.be.eq(timestamp) - res.body.result.files.should.have.length(2) - res.body.result.periodWiseFiles.should.be.eql(periodWiseData) - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]) - done() - }) - }); -}) diff --git a/api-service/src/v1/test/Config.ts b/api-service/src/v1/test/Config.ts deleted file mode 100644 index c7a01641..00000000 --- a/api-service/src/v1/test/Config.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { routesConfig } from "../configs/RoutesConfig" -import { config as appConfig } from "../configs/Config"; - -const config = { - apiDruidEndPoint: `${routesConfig.query.native_query.path}`, - apiDruidSqlEndPoint: `${routesConfig.query.sql_query.path}`, - apiDatasetIngestEndPoint: `${routesConfig.data_ingest.path}`, - apiDatasetSaveEndPoint: `${routesConfig.config.dataset.save.path}`, - apiDatasetUpdateEndPoint: `${routesConfig.config.dataset.update.path}`, - apiDatasetReadEndPoint: `${routesConfig.config.dataset.read.path}`, - apiDatasetListEndPoint: `${routesConfig.config.dataset.list.path}`, - apiDatasourceSaveEndPoint: `${routesConfig.config.datasource.save.path}`, - apiDatasourceUpdateEndPoint: `${routesConfig.config.datasource.update.path}`, - apiDatasourceReadEndPoint: `${routesConfig.config.datasource.read.path}`, - apiDatasourceListEndPoint: `${routesConfig.config.datasource.list.path}`, - apiDatasetSourceConfigSaveEndPoint: `${routesConfig.config.dataset_source_config.save.path}`, - apiDatasetSourceConfigUpdateEndPoint: `${routesConfig.config.dataset_source_config.update.path}`, - apiDatasetSourceConfigReadEndPoint: `${routesConfig.config.dataset_source_config.read.path}`, - apiDatasetSourceConfigListEndPoint: `${routesConfig.config.dataset_source_config.list.path}`, - apiExhaustEndPoint: `${routesConfig.exhaust.path}`, - druidHost: `${appConfig.query_api.druid.host}`, - druidPort: `${appConfig.query_api.druid.port}`, - druidEndPoint: `${appConfig.query_api.druid.native_query_path}`, - druidSqlEndPoint: `${appConfig.query_api.druid.sql_query_path}`, - storage_url_expiry: 3600, - exhaustMaxDateRange: 31, - druidDatasourcesEndPoint: `${appConfig.query_api.druid.list_datasources_path}`, - druidSubmitIngestionEndPoint: `/${appConfig.query_api.druid.submit_ingestion}`, - apiSubmitIngestionEndPoint: `${routesConfig.submit_ingestion.path}` -}; -export { config }; diff --git a/api-service/src/v1/test/DatasetSourceConfigTestService.spec.ts b/api-service/src/v1/test/DatasetSourceConfigTestService.spec.ts deleted file mode 100644 index bcf11ac1..00000000 --- a/api-service/src/v1/test/DatasetSourceConfigTestService.spec.ts +++ /dev/null @@ -1,300 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import constants from '../resources/Constants.json' -import { TestDatasetSourceConfig } from "./Fixtures"; -import { config } from "./Config"; -import { routesConfig } from "../configs/RoutesConfig"; -import { dbConnector } from "../routes/Router"; -import { describe, it } from 'mocha'; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { DatasetStatus } from "../models/DatasetModels"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe("Dataset source config create API", () => { - it("should insert a record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai.spy.on(ResponseHandler, "successResponse", ()=>{ - throw new Error("Error occured while sending response") - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - chai.spy.restore(ResponseHandler, "successResponse") - done(); - }); - }); - it("should not insert record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record if already exists in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([{}]) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.CONFLICT); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["409_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record when request object contains missing fields", (done) => { - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.MISSING_REQUIRED_FIELDS_CREATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) - it("should not insert record when given invalid schema", (done) => { - chai - .request(app) - .post(config.apiDatasetSourceConfigSaveEndPoint) - .send(TestDatasetSourceConfig.INVALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) -}) -describe("Dataset source config update API", () => { - it("should successfully update records in database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve() - }) - chai - .request(app) - .patch(config.apiDatasetSourceConfigUpdateEndPoint) - .send(TestDatasetSourceConfig.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should not update records in database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .patch(config.apiDatasetSourceConfigUpdateEndPoint) - .send(TestDatasetSourceConfig.VALID_UPDATE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.update.api_id) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should not update records when request object does not contain required fields", (done) => { - chai - .request(app) - .patch(config.apiDatasetSourceConfigUpdateEndPoint) - .send(TestDatasetSourceConfig.MISSING_REQUIRED_FIELDS_UPDATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); -}) -describe("Dataset source config read API", () => { - it("should successfully retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([TestDatasetSourceConfig.VALID_RECORD]) - }) - chai - .request(app) - .get(config.apiDatasetSourceConfigReadEndPoint.replace(":datasetId", TestDatasetSourceConfig.SAMPLE_ID).concat(`?status = ${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should throw error if retrieved record is empty", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .get(config.apiDatasetSourceConfigReadEndPoint.replace(":datasetId", TestDatasetSourceConfig.SAMPLE_ID).concat('?status=DISABLED')) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should not retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .get(config.apiDatasetSourceConfigReadEndPoint.replace(":datasetId", TestDatasetSourceConfig.SAMPLE_ID).concat('?status=DISABLED')) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - -}) -describe("Dataset source config list API", () => { - it("should successfully list records in the table", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([{}, {}, {}]) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigListEndPoint) - .send(TestDatasetSourceConfig.VALID_LIST_REQUEST_ACTIVE_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("array") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }) - it("should not list records if request object is invalid", (done) => { - chai - .request(app) - .post(config.apiDatasetSourceConfigListEndPoint) - .send({"filters": true}) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) - it("should not list records", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasetSourceConfigListEndPoint) - .send(TestDatasetSourceConfig.VALID_LIST_REQUEST_DISABLED_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset_source_config.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - -}) diff --git a/api-service/src/v1/test/DatasetTestService.spec.ts b/api-service/src/v1/test/DatasetTestService.spec.ts deleted file mode 100644 index 5b4ab241..00000000 --- a/api-service/src/v1/test/DatasetTestService.spec.ts +++ /dev/null @@ -1,366 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import constants from '../resources/Constants.json' -import { TestDataset } from "./Fixtures"; -import { config } from "./Config"; -import { routesConfig } from "../configs/RoutesConfig"; -import { dbConnector } from "../routes/Router"; -import { Datasets } from "../helpers/Datasets"; -import { describe, it } from 'mocha'; -import { DatasetStatus } from "../models/DatasetModels"; - - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe("Dataset create API", () => { - it("should insert a record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should insert a master dataset record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_SCHEMA_MASTER_DATASET) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record if already exists in the database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([{}]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.CONFLICT); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["409_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record when request object contains missing fields", (done) => { - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.MISSING_REQUIRED_FIELDS_CREATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result") - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); - it("should not insert record when given invalid schema", (done) => { - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.INVALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); - it("should not insert the record when there's a duplicate denorm out field", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.DUPLICATE_DENORM_OUT_FIELD) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should insert the record when there's no duplicate denorm out field", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasetSaveEndPoint) - .send(TestDataset.VALID_DENORM_OUT_FIELD) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); -}) -describe("Dataset update API", () => { - beforeEach(() => { - chai.spy.on(dbConnector, "listRecords", () => { - Promise.resolve() - }) - }) - afterEach(() => { - chai.spy.restore(dbConnector, "listRecords") - }) - it("should successfully update records in database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve() - }) - chai - .request(app) - .patch(config.apiDatasetUpdateEndPoint) - .send(TestDataset.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should not update records in database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .patch(config.apiDatasetUpdateEndPoint) - .send(TestDataset.VALID_UPDATE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.update.api_id) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should not update records when request object does not contain required fields", (done) => { - chai - .request(app) - .patch(config.apiDatasetUpdateEndPoint) - .send(TestDataset.MISSING_REQUIRED_FIELDS_UPDATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); -}) -describe("Dataset read API", () => { - it("should successfully retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([TestDataset.VALID_RECORD]) - }) - chai - .request(app) - .get(config.apiDatasetReadEndPoint.replace(":datasetId", TestDataset.SAMPLE_ID).concat(`?status = ${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should throw error if retrieved record is empty", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .get(config.apiDatasetReadEndPoint.replace(":datasetId", TestDataset.SAMPLE_ID).concat('?status=DISABLED')) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should not retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .get(config.apiDatasetReadEndPoint.replace(":datasetId", TestDataset.SAMPLE_ID).concat('?status=DISABLED')) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - -}) -describe("Dataset list API", () => { - it("should successfully list records in the table", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([{}, {}, {}]) - }) - chai - .request(app) - .post(config.apiDatasetListEndPoint) - .send(TestDataset.VALID_LIST_REQUEST_ACTIVE_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("array") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }) - it("should not list records if request object is invalid", (done) => { - chai - .request(app) - .post(config.apiDatasetListEndPoint) - .send({ - "filters": true - }) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) - it("should not list records", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasetListEndPoint) - .send(TestDataset.VALID_LIST_REQUEST_DISABLED_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) -}) - -describe("Error scenarios in Dataset API", () => { - it("should not update records in database", (done) => { - chai.spy.on(Datasets.prototype, "getValues", () => { - throw new Error("error occured while parsing data") - }) - chai - .request(app) - .patch(config.apiDatasetUpdateEndPoint) - .send(TestDataset.VALID_UPDATE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.dataset.update.api_id) - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(Datasets.prototype, "getValues") - done() - }) - }) -}) diff --git a/api-service/src/v1/test/DatasourceTestService.spec.ts b/api-service/src/v1/test/DatasourceTestService.spec.ts deleted file mode 100644 index e6df8718..00000000 --- a/api-service/src/v1/test/DatasourceTestService.spec.ts +++ /dev/null @@ -1,493 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import constants from '../resources/Constants.json' -import { TestDataSource } from "./Fixtures"; -import { config } from "./Config"; -import { routesConfig } from "../configs/RoutesConfig"; -import { dbConnector } from "../routes/Router"; -import { describe, it } from 'mocha'; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { ingestorService } from "../routes/Router"; -import { DatasetStatus } from "../models/DatasetModels"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe('Datasource APIS', () => { - afterEach(()=>{ - chai.spy.restore() - }) - describe("Datasource create API", () => { - it("should insert a record in the database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.message.should.be.eq(constants.RECORD_SAVED) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error when datasource ref is invalid", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.INVALID_DATASOURCE_REF) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error when topic is invalid", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.INVALID_INPUT_TOPIC) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error when dataset id does not exists", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return {} - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "404_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert a record when it already exists in database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([ {} ]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - - res.should.have.status(httpStatus.CONFLICT); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "409_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should throw error", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai.spy.on(ResponseHandler, "successResponse", () => { - throw new Error("Error occured while sending response") - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - chai.spy.restore(ResponseHandler, "successResponse") - done(); - }); - }); - it("should not insert record in the database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error occured while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }); - it("should not insert record when request object contains missing fields", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.MISSING_REQUIRED_FIELDS_CREATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - done(); - }); - }) - it("should not insert record when given invalid schema", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.INVALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - done(); - }); - }) - it("should create datalake datasource", (done)=>{ - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post(config.apiDatasourceSaveEndPoint) - .send(TestDataSource.VALID_DATALAKE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.save.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.message.should.be.eq(constants.RECORD_SAVED) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }) - }) - describe("Datasource update API", () => { - it("should not update records in database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error occured while connecting to postgres")) - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.VALID_UPDATE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - it("should successfully update records in database", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.VALID_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.message.should.be.eq(constants.RECORD_UPDATED) - chai.spy.restore(ingestorService, "getDatasetConfig") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }); - - it("should not update records when request object does not contain required fields", (done) => { - chai.spy.on(ingestorService, "getDatasetConfig", () => { - return { "id": ":telemetry", "dataset_config": { "entry_topic": "telemetry" }, "router_config": { "topic": "telemetry" } } - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.MISSING_REQUIRED_FIELDS_UPDATE) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(ingestorService, "getDatasetConfig") - done(); - }); - }); - it("should update datalake datasource", (done)=>{ - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.VALID_DATALAKE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.message.should.be.eq(constants.RECORD_UPDATED) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }) - it("should not update records when given invalid schema in case of datalake", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .patch(config.apiDatasourceUpdateEndPoint) - .send(TestDataSource.INVALID_DATALAKE_SCHEMA) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.update.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute"); - done(); - }); - }) - }) - describe("Datasource read API", () => { - it("should successfully retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([ TestDataSource.VALID_RECORD ]) - }) - chai - .request(app) - .get(config.apiDatasourceReadEndPoint.replace(":datasourceId", TestDataSource.SAMPLE_ID).concat(`?status=${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should successfully retrieve records from database", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([ TestDataSource.VALID_RECORD ]) - }) - chai - .request(app) - .get(config.apiDatasourceReadEndPoint.replace(":datasourceId", TestDataSource.SAMPLE_ID)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("object") - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should throw error if records are empty", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .get(config.apiDatasourceReadEndPoint.replace(":datasourceId", TestDataSource.SAMPLE_ID).concat(`?status=${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "404_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }), - it("should not retrieve records", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .get(config.apiDatasourceReadEndPoint.replace(":datasourceId", TestDataSource.SAMPLE_ID).concat(`?status=${DatasetStatus.Live}`)) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.read.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - - }) - describe("Datasource list API", () => { - it("should successfully list records in the table", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.resolve([ {} ]) - }) - chai - .request(app) - .post(config.apiDatasourceListEndPoint) - .send(TestDataSource.VALID_LIST_REQUEST_DISABLED_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - res.body.result.should.be.a("array") - chai.spy.restore(dbConnector, "execute") - done(); - }); - }) - it("should not list records if request object is invalid", (done) => { - chai - .request(app) - .post(config.apiDatasourceListEndPoint) - .send({ "filters": true }) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }) - it("should not list records", (done) => { - chai.spy.on(dbConnector, "execute", () => { - return Promise.reject(new Error("error while connecting to postgres")) - }) - chai - .request(app) - .post(config.apiDatasourceListEndPoint) - .send(TestDataSource.VALID_LIST_REQUEST_ACTIVE_STATUS) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.config.datasource.list.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "execute") - done(); - }) - }) - }) -}) - diff --git a/api-service/src/v1/test/Fixtures.ts b/api-service/src/v1/test/Fixtures.ts deleted file mode 100644 index 634db55b..00000000 --- a/api-service/src/v1/test/Fixtures.ts +++ /dev/null @@ -1,154 +0,0 @@ -import { DatasetStatus } from "../models/DatasetModels"; - -class TestDruidQuery { - public static VALID_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"timeseries","dataSource":"telemetry-events","aggregations":[{"type":"count","name":"count"}],"granularity":"all","postAggregations":[],"intervals": "2021-02-19/2021-02-20"}}'; - public static HIGH_DATE_RANGE_GIVEN_AS_LIST = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"groupBy","dataSource":"telemetry-events","dimensions":["actor_type","content_framework"],"limit":15, "metric":"count","granularity":"all","intervals":["2021-01-02/2021-02-05"],"aggregations":[{"type":"count","name":"count"}]}}'; - public static HIGH_DATE_RANGE_GIVEN_AS_STRING = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"topN","dataSource":"telemetry-events","dimension":"actor_id","threshold":10,"metric":"count","granularity":"all","intervals":"2020-12-30/2021-02-02","aggregations":[{"type":"count","name":"count"}]}}'; - public static HIGH_THRESHOLD_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"scan","dataSource":"telemetry-events","dimension":"mid","threshold":1000,"metric":"count","granularity":"all","intervals":["2020-12-31/2021-01-01"],"aggregations":[]}}'; - public static HIGH_LIMIT_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"scan","dataSource":"telemetry-events","granularity":"all","intervals":["2020-12-21/2021-01-01"],"resultFormat":"compactedList","limit":1000,"columns":["actor_id", "mid"],"metrics":{"type":"numeric","metric":"count"},"aggregations":[{"type":"count","name":"count"}]}}'; - public static WITHOUT_THRESOLD_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"timeBoundary","dataSource":"telemetry-events","dimension":"content_status","metric":"count","granularity":"all","intervals":["2020-12-21/2020-12-22"],"aggregations":[]}}'; - public static WITHOUT_DATE_RANGE_QUERY = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"search","dataSource":"telemetry-events","granularity":"all","intervals":"","resultFormat":"compactedList","columns":["__time"],"metrics":{"type":"numeric","metric":"count"},"aggregations":[{"type":"count","name":"count"}]}}'; - public static UNSUPPORTED_DATA_SOURCE = - '{"context":{"dataSource":"invalid_data_source"},"query":{"queryType":"timeBoundary","dataSource":"invalid_data_source","granularity":"all","intervals":["2022-10-17/2022-10-19"],"resultFormat":"compactedList","columns":["__time","scans"],"metrics":{"type":"numeric","metric":"count"},"aggregations":[{"type":"count","name":"count"}]}}'; - public static INVALID_QUERY_TYPE = - '{"context":{"dataSource":"telemetry-events"},"query":{"queryType":"invalidQueryType", "dataSource":"telemetry-events", "granularity":"all", "intervals":"2021-12-31/2022-01-20"}}'; - public static UNSUPPORTED_SCHEMA = - '{"context":{},"query":{"queryType":"invalidQueryType", "dataSource":"telemetry-events", "granularity":"all", "intervals":"2021-12-31/2022-01-20"}}'; - public static VALID_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT * FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}}'; - public static VALID_QUERY_DATALAKE = - '{"context":{"dataSource":"telemetry-events", "dataSourceType":"datalake"},"querySql":{"query": "SELECT COUNT(*) AS totalRatingsCount FROM \\"d1\\" LIMIT 100"}}'; - public static INVALID_DATASOURCE_TYPE = '{"context":{"dataSource":"telemetry-events", "dataSourceType":"random"},"querySql":{"query": "SELECT COUNT(*) AS totalRatingsCount FROM \\"d1\\" LIMIT 100"}}'; - public static EMPTY_SQL_QUERY = '{"context":{"dataSource":"telemetry-events", "dataSourceType":"datalake"},"querySql":{"query": ""}}'; - public static CASE_INSENSITIVE_SQL_QUERY = '{"context":{"dataSource":"telemetry-events","dataSourceType":"datalake"},"querySql":{"query":"select * from \\"telemetry-events\\" where __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}}'; - public static HIGH_LIMIT_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT mid FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-22\' LIMIT 1000"}}'; - public static WITHOUT_LIMIT_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT actor_type, content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-02\'"}}'; - public static HIGH_DATE_RANGE_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT actor_type, content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-02-12\' LIMIT 10"}}'; - public static WITHOUT_DATE_RANGE_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT content_status FROM \\"telemetry-events\\" LIMIT 5"}}'; - public static UNSUPPORTED_DATASOURCE_SQL_QUERY = - '{"context":{"dataSource":"telemetry-events"},"querySql":{"query":"SELECT __time FROM \\"invalid-datasource\\" LIMIT 10"}}'; - public static SKIP_VALIDATION_NATIVE = '{"context":{"dataSource":"system-stats"},"query":{"queryType":"timeBoundary","dataSource":"system-stats","granularity":"all","intervals":["2022-10-17/2022-10-19"],"resultFormat":"compactedList","columns":["__time","scans"],"metrics":{"type":"numeric","metric":"count"},"aggregations":[{"type":"count","name":"count"}]}}'; - public static SKIP_VALIDATION_SQL = '{"context":{"dataSource":"failed-events-summary"},"querySql":{"query":"SELECT * FROM \\"failed-events-summary\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}}'; - public static INVALID_SQL_QUERY = '{\"context\":{\"dataSource\":\"system-events\",\"granularity\":\"day\"},\"querySql\":{\"query\":\"SELECT * \"}}'; - public static MISSING_TABLE_NAME = '{\"context\":{\"dataSource\":\"system-events\",\"granularity\":\"day\"},\"querySql\":{\"query\":\"SELECT * FROM \"}}'; -} - -class TestDataIngestion { - public static SAMPLE_INDIVIDUAL_EVENT = { "data": { "event": { "context": { "transaction_id": "3d3bac46-d252-4da0-9290-afdd524d0214", "country": "IND", "bpp_id": "becknify.humbhionline.in.mobility.BPP/beckn_open/app1-succinct-in", "city": "std:080", "message_id": "52dcf5a9-8986-47ff-a9d0-f380b23e3dfe", "core_version": "0.9.1", "ttl": "PT1M", "bap_id": "mobilityreferencebap.becknprotocol.io", "domain": "nic2004:60221", "bpp_uri": "https://becknify.humbhionline.in/mobility/beckn_open/app1-succinct-in/bpp", "action": "on_status", "bap_uri": "https://mobilityreferencebap.becknprotocol.io", "timestamp": "2023-02-22T19:06:27.887Z" }, "message": { "order": { "quote": { "breakup": [ { "price": { "currency": "INR", "value": "58.2936244525222" }, "type": "item", "title": "Fare" }, { "price": { "currency": "INR", "value": "10.492852401453995" }, "type": "item", "title": "Tax" } ], "price": { "currency": "INR", "value": "68.7864768539762" } }, "provider": { "locations": [ { "gps": "12.973437,77.608771", "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.provider_location" } ], "id": "./mobility/ind.blr/7@taxi.becknprotocol.io.provider", "descriptor": { "images": [ "https://taxi.becknprotocol.io/companies/view/7" ], "name": "Best Taxies" }, "categories": [ { "id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "descriptor": { "name": "Premium Taxi" } } ], "items": [ { "category_id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "price": { "currency": "INR", "value": "68.7864768539762" }, "descriptor": { "images": [ "https://taxi.becknprotocol.io/resources/images/car.png" ], "code": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi", "name": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi" }, "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.item", "fulfillment_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "tags": { "NameOfModel": "Brezza", "VehicleType": "Premium Taxi", "Make": "Maruti", "FuelType": "Diesel" } } ] }, "id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.order", "state": "Awaiting Driver acceptance", "fulfillment": { "agent": { "phone": "+919082233441", "name": "Michel MJ" }, "start": { "location": { "gps": "12.973437,77.608771" } }, "end": { "location": { "gps": "12.935193,77.624481" } }, "id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "vehicle": { "registration": "KA 05 3456" }, "customer": { "person": { "name": "./Rajat/Mr./Rajat/ /Kumar/" }, "contact": { "phone": "+919867654322", "email": "er.rjtkumar@gmail.com" } } }, "items": [ { "category_id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "price": { "currency": "INR", "value": "68.7864768539762" }, "descriptor": { "images": [ "https://taxi.becknprotocol.io/resources/images/car.png" ], "code": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi", "name": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi" }, "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.item", "fulfillment_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "tags": { "NameOfModel": "Brezza", "VehicleType": "Premium Taxi", "Make": "Maruti", "FuelType": "Diesel" } } ], "billing": { "address": { "country": "IND", "door": "MBT", "city": "std:080", "area_code": "560078", "name": "RajatKumar", "locality": "", "building": ",A33" }, "phone": "+919867654322", "name": "./Rajat/Mr./Rajat/ /Kumar/", "email": "er.rjtkumar@gmail.com" } } } } }} - public static SAMPLE_INPUT = { "data": { "id": "beckn-batch-1", "events": [ { "context": { "transaction_id": "3d3bac46-d252-4da0-9290-afdd524d0214", "country": "IND", "bpp_id": "becknify.humbhionline.in.mobility.BPP/beckn_open/app1-succinct-in", "city": "std:080", "message_id": "52dcf5a9-8986-47ff-a9d0-f380b23e3dfe", "core_version": "0.9.1", "ttl": "PT1M", "bap_id": "mobilityreferencebap.becknprotocol.io", "domain": "nic2004:60221", "bpp_uri": "https://becknify.humbhionline.in/mobility/beckn_open/app1-succinct-in/bpp", "action": "on_status", "bap_uri": "https://mobilityreferencebap.becknprotocol.io", "timestamp": "2023-02-22T19:06:27.887Z" }, "message": { "order": { "quote": { "breakup": [ { "price": { "currency": "INR", "value": "58.2936244525222" }, "type": "item", "title": "Fare" }, { "price": { "currency": "INR", "value": "10.492852401453995" }, "type": "item", "title": "Tax" } ], "price": { "currency": "INR", "value": "68.7864768539762" } }, "provider": { "locations": [ { "gps": "12.973437,77.608771", "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.provider_location" } ], "id": "./mobility/ind.blr/7@taxi.becknprotocol.io.provider", "descriptor": { "images": [ "https://taxi.becknprotocol.io/companies/view/7" ], "name": "Best Taxies" }, "categories": [ { "id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "descriptor": { "name": "Premium Taxi" } } ], "items": [ { "category_id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "price": { "currency": "INR", "value": "68.7864768539762" }, "descriptor": { "images": [ "https://taxi.becknprotocol.io/resources/images/car.png" ], "code": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi", "name": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi" }, "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.item", "fulfillment_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "tags": { "NameOfModel": "Brezza", "VehicleType": "Premium Taxi", "Make": "Maruti", "FuelType": "Diesel" } } ] }, "id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.order", "state": "Awaiting Driver acceptance", "fulfillment": { "agent": { "phone": "+919082233441", "name": "Michel MJ" }, "start": { "location": { "gps": "12.973437,77.608771" } }, "end": { "location": { "gps": "12.935193,77.624481" } }, "id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "vehicle": { "registration": "KA 05 3456" }, "customer": { "person": { "name": "./Rajat/Mr./Rajat/ /Kumar/" }, "contact": { "phone": "+919867654322", "email": "er.rjtkumar@gmail.com" } } }, "items": [ { "category_id": "./mobility/ind.blr/1@taxi.becknprotocol.io.category", "price": { "currency": "INR", "value": "68.7864768539762" }, "descriptor": { "images": [ "https://taxi.becknprotocol.io/resources/images/car.png" ], "code": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi", "name": "Premium Taxi-FuelType:Diesel,Make:Maruti,NameOfModel:Brezza,VehicleType:Premium Taxi" }, "id": "./mobility/ind.blr/17@taxi.becknprotocol.io.item", "fulfillment_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.fulfillment", "tags": { "NameOfModel": "Brezza", "VehicleType": "Premium Taxi", "Make": "Maruti", "FuelType": "Diesel" } } ], "billing": { "address": { "country": "IND", "door": "MBT", "city": "std:080", "area_code": "560078", "name": "RajatKumar", "locality": "", "building": ",A33" }, "phone": "+919867654322", "name": "./Rajat/Mr./Rajat/ /Kumar/", "email": "er.rjtkumar@gmail.com" } } } }, { "context": { "domain": "nic2004:60221", "country": "IND", "city": "std:080", "core_version": "0.9.1", "action": "track", "bap_id": "mobilityreferencebap.becknprotocol.io", "bap_uri": "https://mobilityreferencebap.becknprotocol.io", "bpp_id": "becknify.humbhionline.in.mobility.BPP/beckn_open/app1-succinct-in", "bpp_uri": "https://becknify.humbhionline.in/mobility/beckn_open/app1-succinct-in/bpp", "transaction_id": "3d3bac46-d252-4da0-9290-afdd524d0214", "message_id": "b52878f3-28ed-4c31-8ebb-8989f33c3220", "timestamp": "2023-02-22T19:07:07.887Z", "ttl": "PT1M" }, "message": { "order_id": "./mobility/ind.blr/6285@taxi.becknprotocol.io.order" } } ] } } -} -class TestDataset { - public static VALID_SCHEMA = { "type": "dataset", "dataset_id": "observations", "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "tags": [], "router_config": { "topic": "router.topic" }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" }; - public static VALID_SCHEMA_MASTER_DATASET = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b999c", "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "tags": [], "router_config": { "topic": "router.topic" }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" } - public static VALID_UPDATE_SCHEMA = { "type": "master-dataset", "dataset_id": "observations", "id": "observations", "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "router_config": { "topic": "router.topic" }, "tags": [], "status": DatasetStatus.Retired, "published_date": "2023-03-14T04:46:33.459Z" } - public static INVALID_SCHEMA = { "dataset_id": "observations", "type": "dataset", "name": "observations", "router_config": { "topic": "" }, "data_schema": "string", "dataset_config": { "entry_topic": "local.ingest", "redis_db_host": "localhost", "redis_db_port": 6379 }, "status": DatasetStatus.Live, "published_date": "2023-03-24 12:19:32.091544" } - public static MISSING_REQUIRED_FIELDS_CREATE = { "type": "dataset", "dataset_id": "observations", "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" }; - public static SAMPLE_ID = "observations"; - public static VALID_LIST_REQUEST_ACTIVE_STATUS = { "filters": { "status": [ DatasetStatus.Live ] } }; - public static VALID_LIST_REQUEST_DISABLED_STATUS = { "filters": { "status": [ DatasetStatus.Retired ] } }; - public static MISSING_REQUIRED_FIELDS_UPDATE = { "name": "telemetry-raw", "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "router_config": { "topic": "router.topic" }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" }; - public static VALID_RECORD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b999c", "id": "observations", "name": "telemetry-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": [{ "denorm_key": "", "redis_db": 1, "denorm_out_field": "metadata" }]}, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" }; - public static DUPLICATE_DENORM_OUT_FIELD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b999c", "id": "observations", "name": "telemetry-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": [{ "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata" }, { "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata" }]}, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" }; - public static VALID_DENORM_OUT_FIELD = { "type": "master-dataset", "dataset_id": "3f8b2ba7-9c74-4d7f-8b38-2b0d460b91ac", "id": "observctions", "name": "telemetay-raw", " validation_config": { "validate": true, "mode": "Strict" }, "extraction_config": { "is_batch_event": false, "extraction_key": "", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 } }, "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 3 }, "data_schema": { "type": "object", "properties": { "eid": { "type": "string" }, "ver": { "type": "string" }, "syncts": { "type": "integer" }, "ets": { "type": "integer" }, "mid": { "type": "string" }, "actor": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] }, "edata": { "type": "object", "properties": { "type": { "type": "string" } }, "required": [ "type" ] }, "@timestamp": { "type": "string" }, "context": { "type": "object", "properties": { "pdata": { "type": "object", "properties": { "ver": { "type": "string" }, "id": { "type": "string" }, "pid": { "type": "string" } }, "required": [ "ver", "id", "pid" ] }, "did": { "type": "string" }, "env": { "type": "string" }, "channel": { "type": "string" } }, "required": [ "pdata", "did", "env", "channel" ] }, "@version": { "type": "string" }, "object": { "type": "object", "properties": { "id": { "type": "string" }, "type": { "type": "string" } }, "required": [ "id", "type" ] } }, "required": [ "eid", "ver", "syncts", "ets", "mid", "actor", "edata", "@timestamp", "context", "@version", "object" ] }, "denorm_config": { "redis_db_host": "redis_host", "redis_db_port": "redis_port", "denorm_fields": [{ "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata2" }, { "denorm_key": "test", "redis_db": 1, "denorm_out_field": "metadata" }]}, "tags": [], "router_config": { "topic": "router.topic" }, "client_state": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-13T07:46:06.410Z", "updated_date": "2023-03-14T04:46:33.459Z", "published_date": "2023-03-14T04:46:33.459Z" }; -} - -class TestDataSource { - public static VALID_DATALAKE_SCHEMA = {"id":"sb-telemetry_sb-telemetry","datasource":"sb-telemetry-events","type":"datalake","dataset_id":"sb-telemetry","ingestion_spec":{"dataset":"sb-telemetry-events","schema":{"table":"sb-telemetry-events","partitionColumn":"eid","timestampColumn":"syncts","primaryKey":"mid","columnSpec":[{"type":"string","name":"eid"},{"type":"string","name":"ver"},{"type":"long","name":"syncts"},{"type":"double","name":"ets"},{"type":"boolean","name":"flags_ex_processed"},{"type":"boolean","name":"flags_pp_validation_processed"},{"type":"boolean","name":"flags_pp_duplicate_skipped"},{"type":"boolean","name":"flags_user_denorm"},{"type":"boolean","name":"flags_device_denorm"},{"type":"boolean","name":"flags_loc_denorm"},{"type":"boolean","name":"flags_content_denorm"},{"type":"boolean","name":"flags_coll_denorm"},{"type":"string","name":"mid"},{"type":"string","name":"actor_id"},{"type":"string","name":"actor_type"},{"type":"array","name":"edata_visits"},{"type":"string","name":"edata_type"},{"type":"double","name":"edata_duration"},{"type":"long","name":"edata_size"},{"type":"string","name":"edata_query"},{"type":"array","name":"edata_filters_objectType"},{"type":"array","name":"edata_filters_version"},{"type":"array","name":"edata_filters_status"},{"type":"array","name":"edata_filters_id"},{"type":"boolean","name":"edata_filters_isRootOrg"},{"type":"string","name":"edata_filters_trackable_enabled"},{"type":"json","name":"edata_filters_channel"},{"type":"json","name":"edata_filters_framework"},{"type":"json","name":"edata_filters_resourceType"},{"type":"array","name":"edata_filters_identifier"},{"type":"array","name":"edata_filters_contentType"},{"type":"array","name":"edata_filters_mimeType"},{"type":"array","name":"edata_filters_hashTagId"},{"type":"long","name":"edata_filters_compatibilityLevel_min"},{"type":"long","name":"edata_filters_compatibilityLevel_max"},{"type":"string","name":"edata_filters_createdBy"},{"type":"array","name":"edata_filters_mediaType"},{"type":"string","name":"edata_filters_origin"},{"type":"array","name":"edata_filters_primaryCategory"},{"name":"edata_filters_trackable"},{"type":"string","name":"edata_sort_lastUpdatedOn"},{"type":"array","name":"edata_topn"},{"type":"string","name":"edata_pageid"},{"type":"string","name":"edata_uri"},{"type":"string","name":"edata_subtype"},{"type":"string","name":"edata_id"},{"type":"string","name":"edata_data"},{"type":"string","name":"edata_uaspec_agent"},{"type":"string","name":"edata_uaspec_ver"},{"type":"string","name":"edata_uaspec_system"},{"type":"string","name":"edata_uaspec_platform"},{"type":"string","name":"edata_uaspec_raw"},{"type":"string","name":"edata_state"},{"type":"array","name":"edata_props"},{"type":"string","name":"edata_prevstate"},{"type":"string","name":"edata_dspec_os"},{"type":"string","name":"edata_dspec_make"},{"type":"string","name":"edata_dspec_id"},{"type":"double","name":"edata_dspec_idisk"},{"type":"double","name":"edata_dspec_edisk"},{"type":"double","name":"edata_dspec_scrn"},{"type":"string","name":"edata_dspec_camera"},{"type":"string","name":"edata_dspec_cpu"},{"type":"long","name":"edata_dspec_sims"},{"type":"string","name":"edata_dspec_webview"},{"type":"array","name":"edata_extra_pos"},{"type":"array","name":"edata_extra_values"},{"type":"string","name":"edata_extra_query"},{"type":"string","name":"edata_mode"},{"type":"timestamp","name":"@timestamp"},{"type":"string","name":"context_channel"},{"type":"string","name":"context_pdata_id"},{"type":"string","name":"context_pdata_ver"},{"type":"string","name":"context_pdata_pid"},{"type":"string","name":"context_env"},{"type":"string","name":"context_sid"},{"type":"string","name":"context_rollup_l1"},{"type":"string","name":"context_rollup_l2"},{"type":"string","name":"context_rollup_l3"},{"type":"array","name":"context_cdata"},{"type":"string","name":"context_did"},{"type":"string","name":"context_uid"},{"type":"string","name":"object_id"},{"type":"string","name":"object_type"},{"type":"string","name":"object_ver"},{"type":"string","name":"object_rollup_l1"},{"type":"string","name":"object_version"},{"type":"array","name":"tags"},{"type":"string","name":"obsrv_meta_source_connector"},{"type":"string","name":"obsrv_meta_source_id"}]},"inputFormat":{"type":"json","flattenSpec":{"fields":[{"type":"path","expr":"$.eid","name":"eid"},{"type":"path","expr":"$.ver","name":"ver"},{"type":"path","expr":"$.syncts","name":"syncts"},{"type":"path","expr":"$.ets","name":"ets"},{"type":"path","expr":"$.flags.ex_processed","name":"flags_ex_processed"},{"type":"path","expr":"$.flags.pp_validation_processed","name":"flags_pp_validation_processed"},{"type":"path","expr":"$.flags.pp_duplicate_skipped","name":"flags_pp_duplicate_skipped"},{"type":"path","expr":"$.flags.user_denorm","name":"flags_user_denorm"},{"type":"path","expr":"$.flags.device_denorm","name":"flags_device_denorm"},{"type":"path","expr":"$.flags.loc_denorm","name":"flags_loc_denorm"},{"type":"path","expr":"$.flags.content_denorm","name":"flags_content_denorm"},{"type":"path","expr":"$.flags.coll_denorm","name":"flags_coll_denorm"},{"type":"path","expr":"$.mid","name":"mid"},{"type":"path","expr":"$.actor.id","name":"actor_id"},{"type":"path","expr":"$.actor.type","name":"actor_type"},{"type":"path","expr":"$.edata.visits[*]","name":"edata_visits"},{"type":"path","expr":"$.edata.type","name":"edata_type"},{"type":"path","expr":"$.edata.duration","name":"edata_duration"},{"type":"path","expr":"$.edata.size","name":"edata_size"},{"type":"path","expr":"$.edata.query","name":"edata_query"},{"type":"path","expr":"$.edata.filters.objectType[*]","name":"edata_filters_objectType"},{"type":"path","expr":"$.edata.filters.version[*]","name":"edata_filters_version"},{"type":"path","expr":"$.edata.filters.status[*]","name":"edata_filters_status"},{"type":"path","expr":"$.edata.filters.id[*]","name":"edata_filters_id"},{"type":"path","expr":"$.edata.filters.isRootOrg","name":"edata_filters_isRootOrg"},{"type":"path","expr":"$.edata.filters.trackable.enabled","name":"edata_filters_trackable_enabled"},{"type":"path","expr":"$.edata.filters.channel","name":"edata_filters_channel"},{"type":"path","expr":"$.edata.filters.framework","name":"edata_filters_framework"},{"type":"path","expr":"$.edata.filters.resourceType","name":"edata_filters_resourceType"},{"type":"path","expr":"$.edata.filters.identifier[*]","name":"edata_filters_identifier"},{"type":"path","expr":"$.edata.filters.contentType[*]","name":"edata_filters_contentType"},{"type":"path","expr":"$.edata.filters.mimeType[*]","name":"edata_filters_mimeType"},{"type":"path","expr":"$.edata.filters.hashTagId[*]","name":"edata_filters_hashTagId"},{"type":"path","expr":"$.edata.filters.compatibilityLevel.min","name":"edata_filters_compatibilityLevel_min"},{"type":"path","expr":"$.edata.filters.compatibilityLevel.max","name":"edata_filters_compatibilityLevel_max"},{"type":"path","expr":"$.edata.filters.createdBy","name":"edata_filters_createdBy"},{"type":"path","expr":"$.edata.filters.mediaType[*]","name":"edata_filters_mediaType"},{"type":"path","expr":"$.edata.filters.origin","name":"edata_filters_origin"},{"type":"path","expr":"$.edata.filters.primaryCategory[*]","name":"edata_filters_primaryCategory"},{"type":"path","expr":"$.edata.filters.trackable","name":"edata_filters_trackable"},{"type":"path","expr":"$.edata.sort.lastUpdatedOn","name":"edata_sort_lastUpdatedOn"},{"type":"path","expr":"$.edata.topn[*]","name":"edata_topn"},{"type":"path","expr":"$.edata.pageid","name":"edata_pageid"},{"type":"path","expr":"$.edata.uri","name":"edata_uri"},{"type":"path","expr":"$.edata.subtype","name":"edata_subtype"},{"type":"path","expr":"$.edata.id","name":"edata_id"},{"type":"path","expr":"$.edata.data","name":"edata_data"},{"type":"path","expr":"$.edata.uaspec.agent","name":"edata_uaspec_agent"},{"type":"path","expr":"$.edata.uaspec.ver","name":"edata_uaspec_ver"},{"type":"path","expr":"$.edata.uaspec.system","name":"edata_uaspec_system"},{"type":"path","expr":"$.edata.uaspec.platform","name":"edata_uaspec_platform"},{"type":"path","expr":"$.edata.uaspec.raw","name":"edata_uaspec_raw"},{"type":"path","expr":"$.edata.state","name":"edata_state"},{"type":"path","expr":"$.edata.props[*]","name":"edata_props"},{"type":"path","expr":"$.edata.prevstate","name":"edata_prevstate"},{"type":"path","expr":"$.edata.dspec.os","name":"edata_dspec_os"},{"type":"path","expr":"$.edata.dspec.make","name":"edata_dspec_make"},{"type":"path","expr":"$.edata.dspec.id","name":"edata_dspec_id"},{"type":"path","expr":"$.edata.dspec.idisk","name":"edata_dspec_idisk"},{"type":"path","expr":"$.edata.dspec.edisk","name":"edata_dspec_edisk"},{"type":"path","expr":"$.edata.dspec.scrn","name":"edata_dspec_scrn"},{"type":"path","expr":"$.edata.dspec.camera","name":"edata_dspec_camera"},{"type":"path","expr":"$.edata.dspec.cpu","name":"edata_dspec_cpu"},{"type":"path","expr":"$.edata.dspec.sims","name":"edata_dspec_sims"},{"type":"path","expr":"$.edata.dspec.webview","name":"edata_dspec_webview"},{"type":"path","expr":"$.edata.extra.pos[*]","name":"edata_extra_pos"},{"type":"path","expr":"$.edata.extra.values[*]","name":"edata_extra_values"},{"type":"path","expr":"$.edata.extra.query","name":"edata_extra_query"},{"type":"path","expr":"$.edata.mode","name":"edata_mode"},{"type":"path","expr":"$.@timestamp","name":"@timestamp"},{"type":"path","expr":"$.context.channel","name":"context_channel"},{"type":"path","expr":"$.context.pdata.id","name":"context_pdata_id"},{"type":"path","expr":"$.context.pdata.ver","name":"context_pdata_ver"},{"type":"path","expr":"$.context.pdata.pid","name":"context_pdata_pid"},{"type":"path","expr":"$.context.env","name":"context_env"},{"type":"path","expr":"$.context.sid","name":"context_sid"},{"type":"path","expr":"$.context.rollup.l1","name":"context_rollup_l1"},{"type":"path","expr":"$.context.rollup.l2","name":"context_rollup_l2"},{"type":"path","expr":"$.context.rollup.l3","name":"context_rollup_l3"},{"type":"path","expr":"$.context.cdata[*]","name":"context_cdata"},{"type":"path","expr":"$.context.did","name":"context_did"},{"type":"path","expr":"$.context.uid","name":"context_uid"},{"type":"path","expr":"$.object.id","name":"object_id"},{"type":"path","expr":"$.object.type","name":"object_type"},{"type":"path","expr":"$.object.ver","name":"object_ver"},{"type":"path","expr":"$.object.rollup.l1","name":"object_rollup_l1"},{"type":"path","expr":"$.object.version","name":"object_version"},{"type":"path","expr":"$.tags[*]","name":"tags"},{"type":"path","expr":"$.obsrv_meta.source.connector","name":"obsrv_meta_source_connector"},{"type":"path","expr":"$.obsrv_meta.source.connectorInstance","name":"obsrv_meta_source_id"}]}}},"datasource_ref":"sb-telemetry-events","retention_period":{"enabled":"false"},"archival_policy":{"enabled":"false"},"purge_policy":{"enabled":"false"},"backup_config":{"enabled":"false"},"status":"Live","created_by":"SYSTEM","updated_by":"SYSTEM","published_date":"2023-07-03 00:00:00"} - public static INVALID_DATALAKE_SCHEMA = {"id":"sb-telemetry_sb-telemetry","datasource":"sb-telemetry-events","type":"invalid_type","dataset_id":"sb-telemetry","ingestion_spec":{"dataset":"sb-telemetry-events","schema":{"table":"table_name_doesnt_match_ref","partitionColumn":"eid","timestampColumn":"syncts","primaryKey":"mid","columnSpec":[{"type":"string","name":"eid"},{"type":"string","name":"ver"},{"type":"long","name":"syncts"},{"type":"double","name":"ets"},{"type":"boolean","name":"flags_ex_processed"},{"type":"boolean","name":"flags_pp_validation_processed"},{"type":"boolean","name":"flags_pp_duplicate_skipped"},{"type":"boolean","name":"flags_user_denorm"},{"type":"boolean","name":"flags_device_denorm"},{"type":"boolean","name":"flags_loc_denorm"},{"type":"boolean","name":"flags_content_denorm"},{"type":"boolean","name":"flags_coll_denorm"},{"type":"string","name":"mid"},{"type":"string","name":"actor_id"},{"type":"string","name":"actor_type"},{"type":"array","name":"edata_visits"},{"type":"string","name":"edata_type"},{"type":"double","name":"edata_duration"},{"type":"long","name":"edata_size"},{"type":"string","name":"edata_query"},{"type":"array","name":"edata_filters_objectType"},{"type":"array","name":"edata_filters_version"},{"type":"array","name":"edata_filters_status"},{"type":"array","name":"edata_filters_id"},{"type":"boolean","name":"edata_filters_isRootOrg"},{"type":"string","name":"edata_filters_trackable_enabled"},{"type":"json","name":"edata_filters_channel"},{"type":"json","name":"edata_filters_framework"},{"type":"json","name":"edata_filters_resourceType"},{"type":"array","name":"edata_filters_identifier"},{"type":"array","name":"edata_filters_contentType"},{"type":"array","name":"edata_filters_mimeType"},{"type":"array","name":"edata_filters_hashTagId"},{"type":"long","name":"edata_filters_compatibilityLevel_min"},{"type":"long","name":"edata_filters_compatibilityLevel_max"},{"type":"string","name":"edata_filters_createdBy"},{"type":"array","name":"edata_filters_mediaType"},{"type":"string","name":"edata_filters_origin"},{"type":"array","name":"edata_filters_primaryCategory"},{"name":"edata_filters_trackable"},{"type":"string","name":"edata_sort_lastUpdatedOn"},{"type":"array","name":"edata_topn"},{"type":"string","name":"edata_pageid"},{"type":"string","name":"edata_uri"},{"type":"string","name":"edata_subtype"},{"type":"string","name":"edata_id"},{"type":"string","name":"edata_data"},{"type":"string","name":"edata_uaspec_agent"},{"type":"string","name":"edata_uaspec_ver"},{"type":"string","name":"edata_uaspec_system"},{"type":"string","name":"edata_uaspec_platform"},{"type":"string","name":"edata_uaspec_raw"},{"type":"string","name":"edata_state"},{"type":"array","name":"edata_props"},{"type":"string","name":"edata_prevstate"},{"type":"string","name":"edata_dspec_os"},{"type":"string","name":"edata_dspec_make"},{"type":"string","name":"edata_dspec_id"},{"type":"double","name":"edata_dspec_idisk"},{"type":"double","name":"edata_dspec_edisk"},{"type":"double","name":"edata_dspec_scrn"},{"type":"string","name":"edata_dspec_camera"},{"type":"string","name":"edata_dspec_cpu"},{"type":"long","name":"edata_dspec_sims"},{"type":"string","name":"edata_dspec_webview"},{"type":"array","name":"edata_extra_pos"},{"type":"array","name":"edata_extra_values"},{"type":"string","name":"edata_extra_query"},{"type":"string","name":"edata_mode"},{"type":"timestamp","name":"@timestamp"},{"type":"string","name":"context_channel"},{"type":"string","name":"context_pdata_id"},{"type":"string","name":"context_pdata_ver"},{"type":"string","name":"context_pdata_pid"},{"type":"string","name":"context_env"},{"type":"string","name":"context_sid"},{"type":"string","name":"context_rollup_l1"},{"type":"string","name":"context_rollup_l2"},{"type":"string","name":"context_rollup_l3"},{"type":"array","name":"context_cdata"},{"type":"string","name":"context_did"},{"type":"string","name":"context_uid"},{"type":"string","name":"object_id"},{"type":"string","name":"object_type"},{"type":"string","name":"object_ver"},{"type":"string","name":"object_rollup_l1"},{"type":"string","name":"object_version"},{"type":"array","name":"tags"},{"type":"string","name":"obsrv_meta_source_connector"},{"type":"string","name":"obsrv_meta_source_id"}]},"inputFormat":{"type":"json","flattenSpec":{"fields":[{"type":"path","expr":"$.eid","name":"eid"},{"type":"path","expr":"$.ver","name":"ver"},{"type":"path","expr":"$.syncts","name":"syncts"},{"type":"path","expr":"$.ets","name":"ets"},{"type":"path","expr":"$.flags.ex_processed","name":"flags_ex_processed"},{"type":"path","expr":"$.flags.pp_validation_processed","name":"flags_pp_validation_processed"},{"type":"path","expr":"$.flags.pp_duplicate_skipped","name":"flags_pp_duplicate_skipped"},{"type":"path","expr":"$.flags.user_denorm","name":"flags_user_denorm"},{"type":"path","expr":"$.flags.device_denorm","name":"flags_device_denorm"},{"type":"path","expr":"$.flags.loc_denorm","name":"flags_loc_denorm"},{"type":"path","expr":"$.flags.content_denorm","name":"flags_content_denorm"},{"type":"path","expr":"$.flags.coll_denorm","name":"flags_coll_denorm"},{"type":"path","expr":"$.mid","name":"mid"},{"type":"path","expr":"$.actor.id","name":"actor_id"},{"type":"path","expr":"$.actor.type","name":"actor_type"},{"type":"path","expr":"$.edata.visits[*]","name":"edata_visits"},{"type":"path","expr":"$.edata.type","name":"edata_type"},{"type":"path","expr":"$.edata.duration","name":"edata_duration"},{"type":"path","expr":"$.edata.size","name":"edata_size"},{"type":"path","expr":"$.edata.query","name":"edata_query"},{"type":"path","expr":"$.edata.filters.objectType[*]","name":"edata_filters_objectType"},{"type":"path","expr":"$.edata.filters.version[*]","name":"edata_filters_version"},{"type":"path","expr":"$.edata.filters.status[*]","name":"edata_filters_status"},{"type":"path","expr":"$.edata.filters.id[*]","name":"edata_filters_id"},{"type":"path","expr":"$.edata.filters.isRootOrg","name":"edata_filters_isRootOrg"},{"type":"path","expr":"$.edata.filters.trackable.enabled","name":"edata_filters_trackable_enabled"},{"type":"path","expr":"$.edata.filters.channel","name":"edata_filters_channel"},{"type":"path","expr":"$.edata.filters.framework","name":"edata_filters_framework"},{"type":"path","expr":"$.edata.filters.resourceType","name":"edata_filters_resourceType"},{"type":"path","expr":"$.edata.filters.identifier[*]","name":"edata_filters_identifier"},{"type":"path","expr":"$.edata.filters.contentType[*]","name":"edata_filters_contentType"},{"type":"path","expr":"$.edata.filters.mimeType[*]","name":"edata_filters_mimeType"},{"type":"path","expr":"$.edata.filters.hashTagId[*]","name":"edata_filters_hashTagId"},{"type":"path","expr":"$.edata.filters.compatibilityLevel.min","name":"edata_filters_compatibilityLevel_min"},{"type":"path","expr":"$.edata.filters.compatibilityLevel.max","name":"edata_filters_compatibilityLevel_max"},{"type":"path","expr":"$.edata.filters.createdBy","name":"edata_filters_createdBy"},{"type":"path","expr":"$.edata.filters.mediaType[*]","name":"edata_filters_mediaType"},{"type":"path","expr":"$.edata.filters.origin","name":"edata_filters_origin"},{"type":"path","expr":"$.edata.filters.primaryCategory[*]","name":"edata_filters_primaryCategory"},{"type":"path","expr":"$.edata.filters.trackable","name":"edata_filters_trackable"},{"type":"path","expr":"$.edata.sort.lastUpdatedOn","name":"edata_sort_lastUpdatedOn"},{"type":"path","expr":"$.edata.topn[*]","name":"edata_topn"},{"type":"path","expr":"$.edata.pageid","name":"edata_pageid"},{"type":"path","expr":"$.edata.uri","name":"edata_uri"},{"type":"path","expr":"$.edata.subtype","name":"edata_subtype"},{"type":"path","expr":"$.edata.id","name":"edata_id"},{"type":"path","expr":"$.edata.data","name":"edata_data"},{"type":"path","expr":"$.edata.uaspec.agent","name":"edata_uaspec_agent"},{"type":"path","expr":"$.edata.uaspec.ver","name":"edata_uaspec_ver"},{"type":"path","expr":"$.edata.uaspec.system","name":"edata_uaspec_system"},{"type":"path","expr":"$.edata.uaspec.platform","name":"edata_uaspec_platform"},{"type":"path","expr":"$.edata.uaspec.raw","name":"edata_uaspec_raw"},{"type":"path","expr":"$.edata.state","name":"edata_state"},{"type":"path","expr":"$.edata.props[*]","name":"edata_props"},{"type":"path","expr":"$.edata.prevstate","name":"edata_prevstate"},{"type":"path","expr":"$.edata.dspec.os","name":"edata_dspec_os"},{"type":"path","expr":"$.edata.dspec.make","name":"edata_dspec_make"},{"type":"path","expr":"$.edata.dspec.id","name":"edata_dspec_id"},{"type":"path","expr":"$.edata.dspec.idisk","name":"edata_dspec_idisk"},{"type":"path","expr":"$.edata.dspec.edisk","name":"edata_dspec_edisk"},{"type":"path","expr":"$.edata.dspec.scrn","name":"edata_dspec_scrn"},{"type":"path","expr":"$.edata.dspec.camera","name":"edata_dspec_camera"},{"type":"path","expr":"$.edata.dspec.cpu","name":"edata_dspec_cpu"},{"type":"path","expr":"$.edata.dspec.sims","name":"edata_dspec_sims"},{"type":"path","expr":"$.edata.dspec.webview","name":"edata_dspec_webview"},{"type":"path","expr":"$.edata.extra.pos[*]","name":"edata_extra_pos"},{"type":"path","expr":"$.edata.extra.values[*]","name":"edata_extra_values"},{"type":"path","expr":"$.edata.extra.query","name":"edata_extra_query"},{"type":"path","expr":"$.edata.mode","name":"edata_mode"},{"type":"path","expr":"$.@timestamp","name":"@timestamp"},{"type":"path","expr":"$.context.channel","name":"context_channel"},{"type":"path","expr":"$.context.pdata.id","name":"context_pdata_id"},{"type":"path","expr":"$.context.pdata.ver","name":"context_pdata_ver"},{"type":"path","expr":"$.context.pdata.pid","name":"context_pdata_pid"},{"type":"path","expr":"$.context.env","name":"context_env"},{"type":"path","expr":"$.context.sid","name":"context_sid"},{"type":"path","expr":"$.context.rollup.l1","name":"context_rollup_l1"},{"type":"path","expr":"$.context.rollup.l2","name":"context_rollup_l2"},{"type":"path","expr":"$.context.rollup.l3","name":"context_rollup_l3"},{"type":"path","expr":"$.context.cdata[*]","name":"context_cdata"},{"type":"path","expr":"$.context.did","name":"context_did"},{"type":"path","expr":"$.context.uid","name":"context_uid"},{"type":"path","expr":"$.object.id","name":"object_id"},{"type":"path","expr":"$.object.type","name":"object_type"},{"type":"path","expr":"$.object.ver","name":"object_ver"},{"type":"path","expr":"$.object.rollup.l1","name":"object_rollup_l1"},{"type":"path","expr":"$.object.version","name":"object_version"},{"type":"path","expr":"$.tags[*]","name":"tags"},{"type":"path","expr":"$.obsrv_meta.source.connector","name":"obsrv_meta_source_connector"},{"type":"path","expr":"$.obsrv_meta.source.connectorInstance","name":"obsrv_meta_source_id"}]}}},"datasource_ref":"sb-telemetry-events","retention_period":{"enabled":"false"},"archival_policy":{"enabled":"false"},"purge_policy":{"enabled":"false"},"backup_config":{"enabled":"false"},"status":"Live","created_by":"SYSTEM","updated_by":"SYSTEM","published_date":"2023-07-03 00:00:00"} - public static VALID_SCHEMA = { "dataset_id": "telemetry", "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "telemetry-events", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "fromDate" }, { "type": "string", "name": "toDate" }, { "type": "string", "name": "tes" }, { "type": "string", "name": "uid" }, { "type": "string", "name": "mobile" }, { "type": "string", "name": "ip" }, { "type": "string", "name": "ipv6" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "string", "name": "derivedlocationdata_district" }, { "type": "string", "name": "derivedlocationdata_from" }, { "type": "string", "name": "derivedlocationdata_state" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "type" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "edata_type" }, { "type": "string", "name": "edata_query" }, { "type": "string", "name": "edata_filters_slug" }, { "type": "boolean", "name": "edata_filters_isTenant" }, { "type": "string", "name": "edata_filters_make_type" }, { "type": "string", "name": "edata_topn[*]_id" }, { "name": "edata_items" }, { "type": "array", "name": "userdata_subject" }, { "type": "string", "name": "userdata_district" }, { "type": "string", "name": "userdata_usersubtype" }, { "type": "array", "name": "userdata_grade" }, { "type": "string", "name": "userdata_usersignintype" }, { "type": "string", "name": "userdata_usertype" }, { "type": "string", "name": "userdata_userlogintype" }, { "type": "string", "name": "userdata_state" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "devicedata_statecustomcode" }, { "type": "string", "name": "devicedata_country" }, { "type": "string", "name": "devicedata_iso3166statecode" }, { "type": "string", "name": "devicedata_city" }, { "type": "string", "name": "devicedata_countrycode" }, { "type": "string", "name": "devicedata_state" }, { "type": "string", "name": "devicedata_devicespec_idisk" }, { "type": "string", "name": "devicedata_devicespec_webview" }, { "type": "string", "name": "devicedata_devicespec_os" }, { "type": "string", "name": "devicedata_devicespec_scrn" }, { "type": "string", "name": "devicedata_devicespec_sims" }, { "type": "string", "name": "devicedata_devicespec_cpu" }, { "type": "string", "name": "devicedata_devicespec_id" }, { "type": "string", "name": "devicedata_devicespec_camera" }, { "type": "string", "name": "devicedata_devicespec_edisk" }, { "type": "string", "name": "devicedata_devicespec_make" }, { "type": "string", "name": "devicedata_statecode" }, { "type": "string", "name": "devicedata_districtcustom" }, { "type": "string", "name": "devicedata_statecustomname" }, { "type": "string", "name": "devicedata_userdeclared_district" }, { "type": "string", "name": "devicedata_userdeclared_state" }, { "type": "string", "name": "context_cdata[*]_id" }, { "type": "string", "name": "context_cdata[*]_type" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_version" }, { "type": "string", "name": "ver" } ] }, "timestampSpec": { "column": "arrival-time", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "eid", "fieldName": "eid" }, { "type": "doubleSum", "name": "syncts", "fieldName": "syncts" }, { "type": "doubleSum", "name": "ets", "fieldName": "ets" }, { "type": "doubleSum", "name": "edata_duration", "fieldName": "edata_duration" }, { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_make_vers", "fieldName": "edata_filters_make_vers" }, { "type": "doubleSum", "name": "devicedata_firstaccess", "fieldName": "devicedata_firstaccess" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "HOUR", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxRowsPerSegment": 50000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "telemetry", "consumerProperties": {}, "taskCount": 1, "replicas": 1, "taskDuration": "PT8H", "useEarliestOffset": false, "completionTimeout": "PT8H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.make.vers", "name": "edata_filters_make_vers" }, { "type": "path", "expr": "$.devicedata.firstaccess", "name": "devicedata_firstaccess" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.derivedlocationdata.district", "name": "derivedlocationdata_district" }, { "type": "path", "expr": "$.derivedlocationdata.from", "name": "derivedlocationdata_from" }, { "type": "path", "expr": "$.derivedlocationdata.state", "name": "derivedlocationdata_state" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.type", "name": "type" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.slug", "name": "edata_filters_slug" }, { "type": "path", "expr": "$.edata.filters.isTenant", "name": "edata_filters_isTenant" }, { "type": "path", "expr": "$.edata.filters.make.type", "name": "edata_filters_make_type" }, { "type": "path", "expr": "$.edata.topn[*].id", "name": "edata_topn[*]_id" }, { "type": "path", "expr": "$.edata.items", "name": "edata_items" }, { "type": "path", "expr": "$.userdata.subject[*]", "name": "userdata_subject" }, { "type": "path", "expr": "$.userdata.district", "name": "userdata_district" }, { "type": "path", "expr": "$.userdata.usersubtype", "name": "userdata_usersubtype" }, { "type": "path", "expr": "$.userdata.grade[*]", "name": "userdata_grade" }, { "type": "path", "expr": "$.userdata.usersignintype", "name": "userdata_usersignintype" }, { "type": "path", "expr": "$.userdata.usertype", "name": "userdata_usertype" }, { "type": "path", "expr": "$.userdata.userlogintype", "name": "userdata_userlogintype" }, { "type": "path", "expr": "$.userdata.state", "name": "userdata_state" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.devicedata.statecustomcode", "name": "devicedata_statecustomcode" }, { "type": "path", "expr": "$.devicedata.country", "name": "devicedata_country" }, { "type": "path", "expr": "$.devicedata.iso3166statecode", "name": "devicedata_iso3166statecode" }, { "type": "path", "expr": "$.devicedata.city", "name": "devicedata_city" }, { "type": "path", "expr": "$.devicedata.countrycode", "name": "devicedata_countrycode" }, { "type": "path", "expr": "$.devicedata.state", "name": "devicedata_state" }, { "type": "path", "expr": "$.devicedata.devicespec.idisk", "name": "devicedata_devicespec_idisk" }, { "type": "path", "expr": "$.devicedata.devicespec.webview", "name": "devicedata_devicespec_webview" }, { "type": "path", "expr": "$.devicedata.devicespec.os", "name": "devicedata_devicespec_os" }, { "type": "path", "expr": "$.devicedata.devicespec.scrn", "name": "devicedata_devicespec_scrn" }, { "type": "path", "expr": "$.devicedata.devicespec.sims", "name": "devicedata_devicespec_sims" }, { "type": "path", "expr": "$.devicedata.devicespec.cpu", "name": "devicedata_devicespec_cpu" }, { "type": "path", "expr": "$.devicedata.devicespec.id", "name": "devicedata_devicespec_id" }, { "type": "path", "expr": "$.devicedata.devicespec.camera", "name": "devicedata_devicespec_camera" }, { "type": "path", "expr": "$.devicedata.devicespec.edisk", "name": "devicedata_devicespec_edisk" }, { "type": "path", "expr": "$.devicedata.devicespec.make", "name": "devicedata_devicespec_make" }, { "type": "path", "expr": "$.devicedata.statecode", "name": "devicedata_statecode" }, { "type": "path", "expr": "$.devicedata.districtcustom", "name": "devicedata_districtcustom" }, { "type": "path", "expr": "$.devicedata.statecustomname", "name": "devicedata_statecustomname" }, { "type": "path", "expr": "$.devicedata.userdeclared.district", "name": "devicedata_userdeclared_district" }, { "type": "path", "expr": "$.devicedata.userdeclared.state", "name": "devicedata_userdeclared_state" }, { "type": "path", "expr": "$.context.cdata[*].id", "name": "context_cdata[*]_id" }, { "type": "path", "expr": "$.context.cdata[*].type", "name": "context_cdata[*]_type" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.ver", "name": "ver" } ] } }, "appendToExisting": false } } }, "type":"druid", "datasource": "telemetry-events", "datasource_ref": "telemetry-events", "status": DatasetStatus.Retired, "published_date": "2023-03-14T04:46:33.459Z" }; - public static VALID_UPDATE_SCHEMA = { "id": "telemetry_telemetry-events", "dataset_id": "telemetry", "type":"druid", "datasource": "telemetry-events", "backup_config": { "enabled": true }, "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z", "datasource_ref": "telemetry-events" }; - public static INVALID_SCHEMA = { "dataset_id": "telemetry", "ingestion_spec": "invalid data type", "type":"druid", "datasource": "telemetry-events", "status": DatasetStatus.Retired, "published_date": "2023-03-14T04:46:33.459Z", "datasource_ref": "telemetry-events", }; - public static INVALID_INPUT_TOPIC = { "id": "sb-telemetry_sb-telemetry", "type":"druid", "datasource": "sb-telemetry", "dataset_id": "sb-telemetry", "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "sb-telemetry.1_DAY", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "eid" }, { "type": "string", "name": "ver" }, { "type": "long", "name": "syncts" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "boolean", "name": "flags_content_denorm" }, { "type": "boolean", "name": "flags_coll_denorm" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "edata_type" }, { "type": "long", "name": "edata_duration" }, { "type": "string", "name": "edata_query" }, { "type": "array", "name": "edata_filters_objectType" }, { "type": "array", "name": "edata_filters_version" }, { "type": "array", "name": "edata_filters_status" }, { "type": "array", "name": "edata_filters_id" }, { "type": "boolean", "name": "edata_filters_isRootOrg" }, { "type": "string", "name": "edata_filters_trackable_enabled" }, { "type": "array", "name": "edata_filters_identifier" }, { "type": "array", "name": "edata_filters_contentType" }, { "type": "array", "name": "edata_filters_mimeType" }, { "type": "array", "name": "edata_filters_hashTagId" }, { "type": "string", "name": "edata_filters_createdBy" }, { "type": "array", "name": "edata_filters_mediaType" }, { "type": "string", "name": "edata_filters_origin" }, { "type": "array", "name": "edata_filters_primaryCategory" }, { "name": "edata_filters_trackable" }, { "type": "string", "name": "edata_sort_lastUpdatedOn" }, { "type": "array", "name": "edata_topn" }, { "type": "string", "name": "edata_pageid" }, { "type": "string", "name": "edata_uri" }, { "type": "string", "name": "edata_subtype" }, { "type": "string", "name": "edata_id" }, { "type": "string", "name": "edata_data" }, { "type": "string", "name": "edata_uaspec_agent" }, { "type": "string", "name": "edata_uaspec_ver" }, { "type": "string", "name": "edata_uaspec_system" }, { "type": "string", "name": "edata_uaspec_platform" }, { "type": "string", "name": "edata_uaspec_raw" }, { "type": "string", "name": "edata_state" }, { "type": "array", "name": "edata_props" }, { "type": "string", "name": "edata_prevstate" }, { "type": "string", "name": "edata_dspec_os" }, { "type": "string", "name": "edata_dspec_make" }, { "type": "string", "name": "edata_dspec_id" }, { "type": "long", "name": "edata_dspec_idisk" }, { "type": "long", "name": "edata_dspec_edisk" }, { "type": "long", "name": "edata_dspec_scrn" }, { "type": "string", "name": "edata_dspec_camera" }, { "type": "string", "name": "edata_dspec_cpu" }, { "type": "string", "name": "edata_dspec_webview" }, { "type": "array", "name": "edata_extra_pos" }, { "type": "array", "name": "edata_extra_values" }, { "type": "string", "name": "edata_extra_query" }, { "type": "string", "name": "edata_mode" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "context_rollup_l2" }, { "type": "string", "name": "context_rollup_l3" }, { "type": "array", "name": "context_cdata" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_uid" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_ver" }, { "type": "string", "name": "object_rollup_l1" }, { "type": "string", "name": "object_version" }, { "type": "array", "name": "tags" } ] }, "timestampSpec": { "column": "ets", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_compatibilityLevel_min", "fieldName": "edata_filters_compatibilityLevel_min" }, { "type": "doubleSum", "name": "edata_filters_compatibilityLevel_max", "fieldName": "edata_filters_compatibilityLevel_max" }, { "type": "doubleSum", "name": "edata_dspec_sims", "fieldName": "edata_dspec_sims" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxBytesInMemory": 134217728, "maxRowsPerSegment": 500000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "invalid_topic", "consumerProperties": { "bootstrap.servers": "kafka-headless.kafka.svc:9092" }, "taskCount": 1, "replicas": 1, "taskDuration": "PT1H", "useEarliestOffset": true, "completionTimeout": "PT1H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.ver", "name": "ver" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.flags.content_denorm", "name": "flags_content_denorm" }, { "type": "path", "expr": "$.flags.coll_denorm", "name": "flags_coll_denorm" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.objectType[*]", "name": "edata_filters_objectType" }, { "type": "path", "expr": "$.edata.filters.version[*]", "name": "edata_filters_version" }, { "type": "path", "expr": "$.edata.filters.status[*]", "name": "edata_filters_status" }, { "type": "path", "expr": "$.edata.filters.id[*]", "name": "edata_filters_id" }, { "type": "path", "expr": "$.edata.filters.isRootOrg", "name": "edata_filters_isRootOrg" }, { "type": "path", "expr": "$.edata.filters.trackable.enabled", "name": "edata_filters_trackable_enabled" }, { "type": "path", "expr": "$.edata.filters.identifier[*]", "name": "edata_filters_identifier" }, { "type": "path", "expr": "$.edata.filters.contentType[*]", "name": "edata_filters_contentType" }, { "type": "path", "expr": "$.edata.filters.mimeType[*]", "name": "edata_filters_mimeType" }, { "type": "path", "expr": "$.edata.filters.hashTagId[*]", "name": "edata_filters_hashTagId" }, { "type": "path", "expr": "$.edata.filters.createdBy", "name": "edata_filters_createdBy" }, { "type": "path", "expr": "$.edata.filters.mediaType[*]", "name": "edata_filters_mediaType" }, { "type": "path", "expr": "$.edata.filters.origin", "name": "edata_filters_origin" }, { "type": "path", "expr": "$.edata.filters.primaryCategory[*]", "name": "edata_filters_primaryCategory" }, { "type": "path", "expr": "$.edata.filters.trackable", "name": "edata_filters_trackable" }, { "type": "path", "expr": "$.edata.sort.lastUpdatedOn", "name": "edata_sort_lastUpdatedOn" }, { "type": "path", "expr": "$.edata.topn[*]", "name": "edata_topn" }, { "type": "path", "expr": "$.edata.pageid", "name": "edata_pageid" }, { "type": "path", "expr": "$.edata.uri", "name": "edata_uri" }, { "type": "path", "expr": "$.edata.subtype", "name": "edata_subtype" }, { "type": "path", "expr": "$.edata.id", "name": "edata_id" }, { "type": "path", "expr": "$.edata.data", "name": "edata_data" }, { "type": "path", "expr": "$.edata.uaspec.agent", "name": "edata_uaspec_agent" }, { "type": "path", "expr": "$.edata.uaspec.ver", "name": "edata_uaspec_ver" }, { "type": "path", "expr": "$.edata.uaspec.system", "name": "edata_uaspec_system" }, { "type": "path", "expr": "$.edata.uaspec.platform", "name": "edata_uaspec_platform" }, { "type": "path", "expr": "$.edata.uaspec.raw", "name": "edata_uaspec_raw" }, { "type": "path", "expr": "$.edata.state", "name": "edata_state" }, { "type": "path", "expr": "$.edata.props[*]", "name": "edata_props" }, { "type": "path", "expr": "$.edata.prevstate", "name": "edata_prevstate" }, { "type": "path", "expr": "$.edata.dspec.os", "name": "edata_dspec_os" }, { "type": "path", "expr": "$.edata.dspec.make", "name": "edata_dspec_make" }, { "type": "path", "expr": "$.edata.dspec.id", "name": "edata_dspec_id" }, { "type": "path", "expr": "$.edata.dspec.idisk", "name": "edata_dspec_idisk" }, { "type": "path", "expr": "$.edata.dspec.edisk", "name": "edata_dspec_edisk" }, { "type": "path", "expr": "$.edata.dspec.scrn", "name": "edata_dspec_scrn" }, { "type": "path", "expr": "$.edata.dspec.camera", "name": "edata_dspec_camera" }, { "type": "path", "expr": "$.edata.dspec.cpu", "name": "edata_dspec_cpu" }, { "type": "path", "expr": "$.edata.dspec.webview", "name": "edata_dspec_webview" }, { "type": "path", "expr": "$.edata.extra.pos[*]", "name": "edata_extra_pos" }, { "type": "path", "expr": "$.edata.extra.values[*]", "name": "edata_extra_values" }, { "type": "path", "expr": "$.edata.extra.query", "name": "edata_extra_query" }, { "type": "path", "expr": "$.edata.mode", "name": "edata_mode" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.context.rollup.l2", "name": "context_rollup_l2" }, { "type": "path", "expr": "$.context.rollup.l3", "name": "context_rollup_l3" }, { "type": "path", "expr": "$.context.cdata[*]", "name": "context_cdata" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.uid", "name": "context_uid" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.ver", "name": "object_ver" }, { "type": "path", "expr": "$.object.rollup.l1", "name": "object_rollup_l1" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.tags[*]", "name": "tags" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.compatibilityLevel.min", "name": "edata_filters_compatibilityLevel_min" }, { "type": "path", "expr": "$.edata.filters.compatibilityLevel.max", "name": "edata_filters_compatibilityLevel_max" }, { "type": "path", "expr": "$.edata.dspec.sims", "name": "edata_dspec_sims" } ] } }, "appendToExisting": false } } }, "datasource_ref": "sb-telemetry.1_DAY", "retention_period": { "enabled": "false" }, "archival_policy": { "enabled": "false" }, "purge_policy": { "enabled": "false" }, "backup_config": { "enabled": "false" }, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "published_date": "2023-07-03 00:00:00", "metadata": { "aggregated": false, "granularity": "day" } } - public static INVALID_DATASOURCE_REF = { "id": "sb-telemetry_sb-telemetry", "type":"druid", "datasource": "sb-telemetry", "dataset_id": "sb-telemetry", "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "invalid_datasource", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "eid" }, { "type": "string", "name": "ver" }, { "type": "long", "name": "syncts" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "boolean", "name": "flags_content_denorm" }, { "type": "boolean", "name": "flags_coll_denorm" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "edata_type" }, { "type": "long", "name": "edata_duration" }, { "type": "string", "name": "edata_query" }, { "type": "array", "name": "edata_filters_objectType" }, { "type": "array", "name": "edata_filters_version" }, { "type": "array", "name": "edata_filters_status" }, { "type": "array", "name": "edata_filters_id" }, { "type": "boolean", "name": "edata_filters_isRootOrg" }, { "type": "string", "name": "edata_filters_trackable_enabled" }, { "type": "array", "name": "edata_filters_identifier" }, { "type": "array", "name": "edata_filters_contentType" }, { "type": "array", "name": "edata_filters_mimeType" }, { "type": "array", "name": "edata_filters_hashTagId" }, { "type": "string", "name": "edata_filters_createdBy" }, { "type": "array", "name": "edata_filters_mediaType" }, { "type": "string", "name": "edata_filters_origin" }, { "type": "array", "name": "edata_filters_primaryCategory" }, { "name": "edata_filters_trackable" }, { "type": "string", "name": "edata_sort_lastUpdatedOn" }, { "type": "array", "name": "edata_topn" }, { "type": "string", "name": "edata_pageid" }, { "type": "string", "name": "edata_uri" }, { "type": "string", "name": "edata_subtype" }, { "type": "string", "name": "edata_id" }, { "type": "string", "name": "edata_data" }, { "type": "string", "name": "edata_uaspec_agent" }, { "type": "string", "name": "edata_uaspec_ver" }, { "type": "string", "name": "edata_uaspec_system" }, { "type": "string", "name": "edata_uaspec_platform" }, { "type": "string", "name": "edata_uaspec_raw" }, { "type": "string", "name": "edata_state" }, { "type": "array", "name": "edata_props" }, { "type": "string", "name": "edata_prevstate" }, { "type": "string", "name": "edata_dspec_os" }, { "type": "string", "name": "edata_dspec_make" }, { "type": "string", "name": "edata_dspec_id" }, { "type": "long", "name": "edata_dspec_idisk" }, { "type": "long", "name": "edata_dspec_edisk" }, { "type": "long", "name": "edata_dspec_scrn" }, { "type": "string", "name": "edata_dspec_camera" }, { "type": "string", "name": "edata_dspec_cpu" }, { "type": "string", "name": "edata_dspec_webview" }, { "type": "array", "name": "edata_extra_pos" }, { "type": "array", "name": "edata_extra_values" }, { "type": "string", "name": "edata_extra_query" }, { "type": "string", "name": "edata_mode" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "context_rollup_l2" }, { "type": "string", "name": "context_rollup_l3" }, { "type": "array", "name": "context_cdata" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_uid" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_ver" }, { "type": "string", "name": "object_rollup_l1" }, { "type": "string", "name": "object_version" }, { "type": "array", "name": "tags" } ] }, "timestampSpec": { "column": "ets", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_compatibilityLevel_min", "fieldName": "edata_filters_compatibilityLevel_min" }, { "type": "doubleSum", "name": "edata_filters_compatibilityLevel_max", "fieldName": "edata_filters_compatibilityLevel_max" }, { "type": "doubleSum", "name": "edata_dspec_sims", "fieldName": "edata_dspec_sims" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxBytesInMemory": 134217728, "maxRowsPerSegment": 500000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "invalid_topic", "consumerProperties": { "bootstrap.servers": "kafka-headless.kafka.svc:9092" }, "taskCount": 1, "replicas": 1, "taskDuration": "PT1H", "useEarliestOffset": true, "completionTimeout": "PT1H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.ver", "name": "ver" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.flags.content_denorm", "name": "flags_content_denorm" }, { "type": "path", "expr": "$.flags.coll_denorm", "name": "flags_coll_denorm" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.objectType[*]", "name": "edata_filters_objectType" }, { "type": "path", "expr": "$.edata.filters.version[*]", "name": "edata_filters_version" }, { "type": "path", "expr": "$.edata.filters.status[*]", "name": "edata_filters_status" }, { "type": "path", "expr": "$.edata.filters.id[*]", "name": "edata_filters_id" }, { "type": "path", "expr": "$.edata.filters.isRootOrg", "name": "edata_filters_isRootOrg" }, { "type": "path", "expr": "$.edata.filters.trackable.enabled", "name": "edata_filters_trackable_enabled" }, { "type": "path", "expr": "$.edata.filters.identifier[*]", "name": "edata_filters_identifier" }, { "type": "path", "expr": "$.edata.filters.contentType[*]", "name": "edata_filters_contentType" }, { "type": "path", "expr": "$.edata.filters.mimeType[*]", "name": "edata_filters_mimeType" }, { "type": "path", "expr": "$.edata.filters.hashTagId[*]", "name": "edata_filters_hashTagId" }, { "type": "path", "expr": "$.edata.filters.createdBy", "name": "edata_filters_createdBy" }, { "type": "path", "expr": "$.edata.filters.mediaType[*]", "name": "edata_filters_mediaType" }, { "type": "path", "expr": "$.edata.filters.origin", "name": "edata_filters_origin" }, { "type": "path", "expr": "$.edata.filters.primaryCategory[*]", "name": "edata_filters_primaryCategory" }, { "type": "path", "expr": "$.edata.filters.trackable", "name": "edata_filters_trackable" }, { "type": "path", "expr": "$.edata.sort.lastUpdatedOn", "name": "edata_sort_lastUpdatedOn" }, { "type": "path", "expr": "$.edata.topn[*]", "name": "edata_topn" }, { "type": "path", "expr": "$.edata.pageid", "name": "edata_pageid" }, { "type": "path", "expr": "$.edata.uri", "name": "edata_uri" }, { "type": "path", "expr": "$.edata.subtype", "name": "edata_subtype" }, { "type": "path", "expr": "$.edata.id", "name": "edata_id" }, { "type": "path", "expr": "$.edata.data", "name": "edata_data" }, { "type": "path", "expr": "$.edata.uaspec.agent", "name": "edata_uaspec_agent" }, { "type": "path", "expr": "$.edata.uaspec.ver", "name": "edata_uaspec_ver" }, { "type": "path", "expr": "$.edata.uaspec.system", "name": "edata_uaspec_system" }, { "type": "path", "expr": "$.edata.uaspec.platform", "name": "edata_uaspec_platform" }, { "type": "path", "expr": "$.edata.uaspec.raw", "name": "edata_uaspec_raw" }, { "type": "path", "expr": "$.edata.state", "name": "edata_state" }, { "type": "path", "expr": "$.edata.props[*]", "name": "edata_props" }, { "type": "path", "expr": "$.edata.prevstate", "name": "edata_prevstate" }, { "type": "path", "expr": "$.edata.dspec.os", "name": "edata_dspec_os" }, { "type": "path", "expr": "$.edata.dspec.make", "name": "edata_dspec_make" }, { "type": "path", "expr": "$.edata.dspec.id", "name": "edata_dspec_id" }, { "type": "path", "expr": "$.edata.dspec.idisk", "name": "edata_dspec_idisk" }, { "type": "path", "expr": "$.edata.dspec.edisk", "name": "edata_dspec_edisk" }, { "type": "path", "expr": "$.edata.dspec.scrn", "name": "edata_dspec_scrn" }, { "type": "path", "expr": "$.edata.dspec.camera", "name": "edata_dspec_camera" }, { "type": "path", "expr": "$.edata.dspec.cpu", "name": "edata_dspec_cpu" }, { "type": "path", "expr": "$.edata.dspec.webview", "name": "edata_dspec_webview" }, { "type": "path", "expr": "$.edata.extra.pos[*]", "name": "edata_extra_pos" }, { "type": "path", "expr": "$.edata.extra.values[*]", "name": "edata_extra_values" }, { "type": "path", "expr": "$.edata.extra.query", "name": "edata_extra_query" }, { "type": "path", "expr": "$.edata.mode", "name": "edata_mode" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.context.rollup.l2", "name": "context_rollup_l2" }, { "type": "path", "expr": "$.context.rollup.l3", "name": "context_rollup_l3" }, { "type": "path", "expr": "$.context.cdata[*]", "name": "context_cdata" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.uid", "name": "context_uid" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.ver", "name": "object_ver" }, { "type": "path", "expr": "$.object.rollup.l1", "name": "object_rollup_l1" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.tags[*]", "name": "tags" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.compatibilityLevel.min", "name": "edata_filters_compatibilityLevel_min" }, { "type": "path", "expr": "$.edata.filters.compatibilityLevel.max", "name": "edata_filters_compatibilityLevel_max" }, { "type": "path", "expr": "$.edata.dspec.sims", "name": "edata_dspec_sims" } ] } }, "appendToExisting": false } } }, "datasource_ref": "sb-telemetry.1_DAY", "retention_period": { "enabled": "false" }, "archival_policy": { "enabled": "false" }, "purge_policy": { "enabled": "false" }, "backup_config": { "enabled": "false" }, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "published_date": "2023-07-03 00:00:00", "metadata": { "aggregated": false, "granularity": "day" } } - public static MISSING_REQUIRED_FIELDS_CREATE = { "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "telemetry-events", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "fromDate" }, { "type": "string", "name": "toDate" }, { "type": "string", "name": "tes" }, { "type": "string", "name": "uid" }, { "type": "string", "name": "mobile" }, { "type": "string", "name": "ip" }, { "type": "string", "name": "ipv6" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "string", "name": "derivedlocationdata_district" }, { "type": "string", "name": "derivedlocationdata_from" }, { "type": "string", "name": "derivedlocationdata_state" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "type" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "edata_type" }, { "type": "string", "name": "edata_query" }, { "type": "string", "name": "edata_filters_slug" }, { "type": "boolean", "name": "edata_filters_isTenant" }, { "type": "string", "name": "edata_filters_make_type" }, { "type": "string", "name": "edata_topn[*]_id" }, { "name": "edata_items" }, { "type": "array", "name": "userdata_subject" }, { "type": "string", "name": "userdata_district" }, { "type": "string", "name": "userdata_usersubtype" }, { "type": "array", "name": "userdata_grade" }, { "type": "string", "name": "userdata_usersignintype" }, { "type": "string", "name": "userdata_usertype" }, { "type": "string", "name": "userdata_userlogintype" }, { "type": "string", "name": "userdata_state" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "devicedata_statecustomcode" }, { "type": "string", "name": "devicedata_country" }, { "type": "string", "name": "devicedata_iso3166statecode" }, { "type": "string", "name": "devicedata_city" }, { "type": "string", "name": "devicedata_countrycode" }, { "type": "string", "name": "devicedata_state" }, { "type": "string", "name": "devicedata_devicespec_idisk" }, { "type": "string", "name": "devicedata_devicespec_webview" }, { "type": "string", "name": "devicedata_devicespec_os" }, { "type": "string", "name": "devicedata_devicespec_scrn" }, { "type": "string", "name": "devicedata_devicespec_sims" }, { "type": "string", "name": "devicedata_devicespec_cpu" }, { "type": "string", "name": "devicedata_devicespec_id" }, { "type": "string", "name": "devicedata_devicespec_camera" }, { "type": "string", "name": "devicedata_devicespec_edisk" }, { "type": "string", "name": "devicedata_devicespec_make" }, { "type": "string", "name": "devicedata_statecode" }, { "type": "string", "name": "devicedata_districtcustom" }, { "type": "string", "name": "devicedata_statecustomname" }, { "type": "string", "name": "devicedata_userdeclared_district" }, { "type": "string", "name": "devicedata_userdeclared_state" }, { "type": "string", "name": "context_cdata[*]_id" }, { "type": "string", "name": "context_cdata[*]_type" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_version" }, { "type": "string", "name": "ver" } ] }, "timestampSpec": { "column": "arrival-time", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "eid", "fieldName": "eid" }, { "type": "doubleSum", "name": "syncts", "fieldName": "syncts" }, { "type": "doubleSum", "name": "ets", "fieldName": "ets" }, { "type": "doubleSum", "name": "edata_duration", "fieldName": "edata_duration" }, { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_make_vers", "fieldName": "edata_filters_make_vers" }, { "type": "doubleSum", "name": "devicedata_firstaccess", "fieldName": "devicedata_firstaccess" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "HOUR", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxRowsPerSegment": 50000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "telemetry", "consumerProperties": {}, "taskCount": 1, "replicas": 1, "taskDuration": "PT8H", "useEarliestOffset": false, "completionTimeout": "PT8H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.make.vers", "name": "edata_filters_make_vers" }, { "type": "path", "expr": "$.devicedata.firstaccess", "name": "devicedata_firstaccess" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.derivedlocationdata.district", "name": "derivedlocationdata_district" }, { "type": "path", "expr": "$.derivedlocationdata.from", "name": "derivedlocationdata_from" }, { "type": "path", "expr": "$.derivedlocationdata.state", "name": "derivedlocationdata_state" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.type", "name": "type" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.slug", "name": "edata_filters_slug" }, { "type": "path", "expr": "$.edata.filters.isTenant", "name": "edata_filters_isTenant" }, { "type": "path", "expr": "$.edata.filters.make.type", "name": "edata_filters_make_type" }, { "type": "path", "expr": "$.edata.topn[*].id", "name": "edata_topn[*]_id" }, { "type": "path", "expr": "$.edata.items", "name": "edata_items" }, { "type": "path", "expr": "$.userdata.subject[*]", "name": "userdata_subject" }, { "type": "path", "expr": "$.userdata.district", "name": "userdata_district" }, { "type": "path", "expr": "$.userdata.usersubtype", "name": "userdata_usersubtype" }, { "type": "path", "expr": "$.userdata.grade[*]", "name": "userdata_grade" }, { "type": "path", "expr": "$.userdata.usersignintype", "name": "userdata_usersignintype" }, { "type": "path", "expr": "$.userdata.usertype", "name": "userdata_usertype" }, { "type": "path", "expr": "$.userdata.userlogintype", "name": "userdata_userlogintype" }, { "type": "path", "expr": "$.userdata.state", "name": "userdata_state" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.devicedata.statecustomcode", "name": "devicedata_statecustomcode" }, { "type": "path", "expr": "$.devicedata.country", "name": "devicedata_country" }, { "type": "path", "expr": "$.devicedata.iso3166statecode", "name": "devicedata_iso3166statecode" }, { "type": "path", "expr": "$.devicedata.city", "name": "devicedata_city" }, { "type": "path", "expr": "$.devicedata.countrycode", "name": "devicedata_countrycode" }, { "type": "path", "expr": "$.devicedata.state", "name": "devicedata_state" }, { "type": "path", "expr": "$.devicedata.devicespec.idisk", "name": "devicedata_devicespec_idisk" }, { "type": "path", "expr": "$.devicedata.devicespec.webview", "name": "devicedata_devicespec_webview" }, { "type": "path", "expr": "$.devicedata.devicespec.os", "name": "devicedata_devicespec_os" }, { "type": "path", "expr": "$.devicedata.devicespec.scrn", "name": "devicedata_devicespec_scrn" }, { "type": "path", "expr": "$.devicedata.devicespec.sims", "name": "devicedata_devicespec_sims" }, { "type": "path", "expr": "$.devicedata.devicespec.cpu", "name": "devicedata_devicespec_cpu" }, { "type": "path", "expr": "$.devicedata.devicespec.id", "name": "devicedata_devicespec_id" }, { "type": "path", "expr": "$.devicedata.devicespec.camera", "name": "devicedata_devicespec_camera" }, { "type": "path", "expr": "$.devicedata.devicespec.edisk", "name": "devicedata_devicespec_edisk" }, { "type": "path", "expr": "$.devicedata.devicespec.make", "name": "devicedata_devicespec_make" }, { "type": "path", "expr": "$.devicedata.statecode", "name": "devicedata_statecode" }, { "type": "path", "expr": "$.devicedata.districtcustom", "name": "devicedata_districtcustom" }, { "type": "path", "expr": "$.devicedata.statecustomname", "name": "devicedata_statecustomname" }, { "type": "path", "expr": "$.devicedata.userdeclared.district", "name": "devicedata_userdeclared_district" }, { "type": "path", "expr": "$.devicedata.userdeclared.state", "name": "devicedata_userdeclared_state" }, { "type": "path", "expr": "$.context.cdata[*].id", "name": "context_cdata[*]_id" }, { "type": "path", "expr": "$.context.cdata[*].type", "name": "context_cdata[*]_type" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.ver", "name": "ver" } ] } }, "appendToExisting": false } } }, "type":"druid", "datasource": "telemetry-events", "datasource_ref": "telemetry-events", "status": DatasetStatus.Live, "published_date": "2023-03-14T04:46:33.459Z" };; - public static MISSING_REQUIRED_FIELDS_UPDATE = { "dataset_id": "telemetry", "backup_config": { "enabled": true }, "status": DatasetStatus.Retired, "published_date": "2023-03-14T04:46:33.459Z" };; - public static SAMPLE_ID = "telemetry_telemetry-events"; - public static VALID_LIST_REQUEST_ACTIVE_STATUS = { "filters": { "status": [ DatasetStatus.Live ] } }; - public static VALID_LIST_REQUEST_DISABLED_STATUS = { "filters": { "status": [ DatasetStatus.Retired ] } }; - - public static VALID_RECORD = { "dataset_id": "telemetry", "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "telemetry-events", "dimensionsSpec": { "dimensions": [ { "type": "string", "name": "fromDate" }, { "type": "string", "name": "toDate" }, { "type": "string", "name": "tes" }, { "type": "string", "name": "uid" }, { "type": "string", "name": "mobile" }, { "type": "string", "name": "ip" }, { "type": "string", "name": "ipv6" }, { "type": "boolean", "name": "flags_ex_processed" }, { "type": "boolean", "name": "flags_pp_validation_processed" }, { "type": "boolean", "name": "flags_pp_duplicate_skipped" }, { "type": "boolean", "name": "flags_device_denorm" }, { "type": "boolean", "name": "flags_user_denorm" }, { "type": "boolean", "name": "flags_loc_denorm" }, { "type": "string", "name": "derivedlocationdata_district" }, { "type": "string", "name": "derivedlocationdata_from" }, { "type": "string", "name": "derivedlocationdata_state" }, { "type": "string", "name": "mid" }, { "type": "string", "name": "type" }, { "type": "string", "name": "actor_type" }, { "type": "string", "name": "actor_id" }, { "type": "string", "name": "edata_type" }, { "type": "string", "name": "edata_query" }, { "type": "string", "name": "edata_filters_slug" }, { "type": "boolean", "name": "edata_filters_isTenant" }, { "type": "string", "name": "edata_filters_make_type" }, { "type": "string", "name": "edata_topn[*]_id" }, { "name": "edata_items" }, { "type": "array", "name": "userdata_subject" }, { "type": "string", "name": "userdata_district" }, { "type": "string", "name": "userdata_usersubtype" }, { "type": "array", "name": "userdata_grade" }, { "type": "string", "name": "userdata_usersignintype" }, { "type": "string", "name": "userdata_usertype" }, { "type": "string", "name": "userdata_userlogintype" }, { "type": "string", "name": "userdata_state" }, { "type": "string", "name": "@timestamp" }, { "type": "string", "name": "devicedata_statecustomcode" }, { "type": "string", "name": "devicedata_country" }, { "type": "string", "name": "devicedata_iso3166statecode" }, { "type": "string", "name": "devicedata_city" }, { "type": "string", "name": "devicedata_countrycode" }, { "type": "string", "name": "devicedata_state" }, { "type": "string", "name": "devicedata_devicespec_idisk" }, { "type": "string", "name": "devicedata_devicespec_webview" }, { "type": "string", "name": "devicedata_devicespec_os" }, { "type": "string", "name": "devicedata_devicespec_scrn" }, { "type": "string", "name": "devicedata_devicespec_sims" }, { "type": "string", "name": "devicedata_devicespec_cpu" }, { "type": "string", "name": "devicedata_devicespec_id" }, { "type": "string", "name": "devicedata_devicespec_camera" }, { "type": "string", "name": "devicedata_devicespec_edisk" }, { "type": "string", "name": "devicedata_devicespec_make" }, { "type": "string", "name": "devicedata_statecode" }, { "type": "string", "name": "devicedata_districtcustom" }, { "type": "string", "name": "devicedata_statecustomname" }, { "type": "string", "name": "devicedata_userdeclared_district" }, { "type": "string", "name": "devicedata_userdeclared_state" }, { "type": "string", "name": "context_cdata[*]_id" }, { "type": "string", "name": "context_cdata[*]_type" }, { "type": "string", "name": "context_env" }, { "type": "string", "name": "context_channel" }, { "type": "string", "name": "context_pdata_id" }, { "type": "string", "name": "context_pdata_pid" }, { "type": "string", "name": "context_pdata_ver" }, { "type": "string", "name": "context_sid" }, { "type": "string", "name": "context_did" }, { "type": "string", "name": "context_rollup_l1" }, { "type": "string", "name": "object_id" }, { "type": "string", "name": "object_type" }, { "type": "string", "name": "object_version" }, { "type": "string", "name": "ver" } ] }, "timestampSpec": { "column": "arrival-time", "format": "auto" }, "metricsSpec": [ { "type": "doubleSum", "name": "eid", "fieldName": "eid" }, { "type": "doubleSum", "name": "syncts", "fieldName": "syncts" }, { "type": "doubleSum", "name": "ets", "fieldName": "ets" }, { "type": "doubleSum", "name": "edata_duration", "fieldName": "edata_duration" }, { "type": "doubleSum", "name": "edata_size", "fieldName": "edata_size" }, { "type": "doubleSum", "name": "edata_filters_make_vers", "fieldName": "edata_filters_make_vers" }, { "type": "doubleSum", "name": "devicedata_firstaccess", "fieldName": "devicedata_firstaccess" } ], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "HOUR", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxRowsPerSegment": 50000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "topic": "telemetry", "consumerProperties": {}, "taskCount": 1, "replicas": 1, "taskDuration": "PT8H", "useEarliestOffset": false, "completionTimeout": "PT8H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [ { "type": "path", "expr": "$.eid", "name": "eid" }, { "type": "path", "expr": "$.syncts", "name": "syncts" }, { "type": "path", "expr": "$.ets", "name": "ets" }, { "type": "path", "expr": "$.edata.duration", "name": "edata_duration" }, { "type": "path", "expr": "$.edata.size", "name": "edata_size" }, { "type": "path", "expr": "$.edata.filters.make.vers", "name": "edata_filters_make_vers" }, { "type": "path", "expr": "$.devicedata.firstaccess", "name": "devicedata_firstaccess" }, { "type": "path", "expr": "$.flags.ex_processed", "name": "flags_ex_processed" }, { "type": "path", "expr": "$.flags.pp_validation_processed", "name": "flags_pp_validation_processed" }, { "type": "path", "expr": "$.flags.pp_duplicate_skipped", "name": "flags_pp_duplicate_skipped" }, { "type": "path", "expr": "$.flags.device_denorm", "name": "flags_device_denorm" }, { "type": "path", "expr": "$.flags.user_denorm", "name": "flags_user_denorm" }, { "type": "path", "expr": "$.flags.loc_denorm", "name": "flags_loc_denorm" }, { "type": "path", "expr": "$.derivedlocationdata.district", "name": "derivedlocationdata_district" }, { "type": "path", "expr": "$.derivedlocationdata.from", "name": "derivedlocationdata_from" }, { "type": "path", "expr": "$.derivedlocationdata.state", "name": "derivedlocationdata_state" }, { "type": "path", "expr": "$.mid", "name": "mid" }, { "type": "path", "expr": "$.type", "name": "type" }, { "type": "path", "expr": "$.actor.type", "name": "actor_type" }, { "type": "path", "expr": "$.actor.id", "name": "actor_id" }, { "type": "path", "expr": "$.edata.type", "name": "edata_type" }, { "type": "path", "expr": "$.edata.query", "name": "edata_query" }, { "type": "path", "expr": "$.edata.filters.slug", "name": "edata_filters_slug" }, { "type": "path", "expr": "$.edata.filters.isTenant", "name": "edata_filters_isTenant" }, { "type": "path", "expr": "$.edata.filters.make.type", "name": "edata_filters_make_type" }, { "type": "path", "expr": "$.edata.topn[*].id", "name": "edata_topn[*]_id" }, { "type": "path", "expr": "$.edata.items", "name": "edata_items" }, { "type": "path", "expr": "$.userdata.subject[*]", "name": "userdata_subject" }, { "type": "path", "expr": "$.userdata.district", "name": "userdata_district" }, { "type": "path", "expr": "$.userdata.usersubtype", "name": "userdata_usersubtype" }, { "type": "path", "expr": "$.userdata.grade[*]", "name": "userdata_grade" }, { "type": "path", "expr": "$.userdata.usersignintype", "name": "userdata_usersignintype" }, { "type": "path", "expr": "$.userdata.usertype", "name": "userdata_usertype" }, { "type": "path", "expr": "$.userdata.userlogintype", "name": "userdata_userlogintype" }, { "type": "path", "expr": "$.userdata.state", "name": "userdata_state" }, { "type": "path", "expr": "$.@timestamp", "name": "@timestamp" }, { "type": "path", "expr": "$.devicedata.statecustomcode", "name": "devicedata_statecustomcode" }, { "type": "path", "expr": "$.devicedata.country", "name": "devicedata_country" }, { "type": "path", "expr": "$.devicedata.iso3166statecode", "name": "devicedata_iso3166statecode" }, { "type": "path", "expr": "$.devicedata.city", "name": "devicedata_city" }, { "type": "path", "expr": "$.devicedata.countrycode", "name": "devicedata_countrycode" }, { "type": "path", "expr": "$.devicedata.state", "name": "devicedata_state" }, { "type": "path", "expr": "$.devicedata.devicespec.idisk", "name": "devicedata_devicespec_idisk" }, { "type": "path", "expr": "$.devicedata.devicespec.webview", "name": "devicedata_devicespec_webview" }, { "type": "path", "expr": "$.devicedata.devicespec.os", "name": "devicedata_devicespec_os" }, { "type": "path", "expr": "$.devicedata.devicespec.scrn", "name": "devicedata_devicespec_scrn" }, { "type": "path", "expr": "$.devicedata.devicespec.sims", "name": "devicedata_devicespec_sims" }, { "type": "path", "expr": "$.devicedata.devicespec.cpu", "name": "devicedata_devicespec_cpu" }, { "type": "path", "expr": "$.devicedata.devicespec.id", "name": "devicedata_devicespec_id" }, { "type": "path", "expr": "$.devicedata.devicespec.camera", "name": "devicedata_devicespec_camera" }, { "type": "path", "expr": "$.devicedata.devicespec.edisk", "name": "devicedata_devicespec_edisk" }, { "type": "path", "expr": "$.devicedata.devicespec.make", "name": "devicedata_devicespec_make" }, { "type": "path", "expr": "$.devicedata.statecode", "name": "devicedata_statecode" }, { "type": "path", "expr": "$.devicedata.districtcustom", "name": "devicedata_districtcustom" }, { "type": "path", "expr": "$.devicedata.statecustomname", "name": "devicedata_statecustomname" }, { "type": "path", "expr": "$.devicedata.userdeclared.district", "name": "devicedata_userdeclared_district" }, { "type": "path", "expr": "$.devicedata.userdeclared.state", "name": "devicedata_userdeclared_state" }, { "type": "path", "expr": "$.context.cdata[*].id", "name": "context_cdata[*]_id" }, { "type": "path", "expr": "$.context.cdata[*].type", "name": "context_cdata[*]_type" }, { "type": "path", "expr": "$.context.env", "name": "context_env" }, { "type": "path", "expr": "$.context.channel", "name": "context_channel" }, { "type": "path", "expr": "$.context.pdata.id", "name": "context_pdata_id" }, { "type": "path", "expr": "$.context.pdata.pid", "name": "context_pdata_pid" }, { "type": "path", "expr": "$.context.pdata.ver", "name": "context_pdata_ver" }, { "type": "path", "expr": "$.context.sid", "name": "context_sid" }, { "type": "path", "expr": "$.context.did", "name": "context_did" }, { "type": "path", "expr": "$.context.rollup.l1", "name": "context_rollup_l1" }, { "type": "path", "expr": "$.object.id", "name": "object_id" }, { "type": "path", "expr": "$.object.type", "name": "object_type" }, { "type": "path", "expr": "$.object.version", "name": "object_version" }, { "type": "path", "expr": "$.ver", "name": "ver" } ] } }, "appendToExisting": false } } }, "type":"druid", "datasource": "telemetry-events", "datasource_ref": "telemetry-events", "retention_period": {}, "archival_policy": {}, "purge_policy": {}, "backup_config": {}, "status": DatasetStatus.Live, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-03-15T20:45:04.737Z", "updated_date": "2023-03-15T20:45:04.733Z", "published_date": "2023-03-15T20:45:04.733Z", "metadata": { "aggregated": false, "granularity": "day" } } - -} - -class TestDatasetSourceConfig { - public static VALID_SCHEMA = { "connector_type": "kafka", "dataset_id": "observations", "status": DatasetStatus.Live } - public static VALID_UPDATE_SCHEMA = { "id": "observations_kafka", "connector_type": "kafka", "dataset_id": "observations", "status": DatasetStatus.Retired } - public static INVALID_SCHEMA = { "connector_type": "kafka", "dataset_id": "observations", "status": {} } - public static MISSING_REQUIRED_FIELDS_CREATE = { "connector_type": "kafka", "status": DatasetStatus.Live } - public static SAMPLE_ID = "observations_kafka"; - public static VALID_LIST_REQUEST_ACTIVE_STATUS = { "filters": { "status": [ DatasetStatus.Live ] } }; - public static VALID_LIST_REQUEST_DISABLED_STATUS = { "filters": { "status": [ DatasetStatus.Retired ] } }; - public static MISSING_REQUIRED_FIELDS_UPDATE = { "connector_type": "kafka", "status": DatasetStatus.Live } - public static VALID_RECORD = { "connector_type": "kafka", "dataset_id": "observations", "connector_config": {}, "status": DatasetStatus.Live, "connector_stats": {}, "created_by": "SYSTEM", "updated_by": "SYSTEM", "created_date": "2023-04-07T18:30:00.000Z", "updated_date": "2023-04-07T18:30:00.000Z" } -} - -class TestExhaust { - public static INVALID_DATE_RANGE = { "from": "20213-06-21", "to": "1234-12-12", "type": "transformed" }; - public static DATE_RANGE_OVER_LIMIT = { "from": "2023-06-01", "to": "2023-07-03", "type": "transformed" }; - public static VALID_REQUEST = { "from": "2023-06-01", "to": "2023-06-30", "type": "transformed" }; -} - - class TestSubmitIngestion { - public static VALID_INGESTION_SPEC = { - "type": "kafka", - "spec": { - "dataSchema": { - "dataSource": "obsrv-telemetry-events", - "dimensionsSpec": { - "dimensions": [] - }, - "timestampSpec": { - "column": "arrival-time", - "format": "auto" - }, - "metricsSpec": [], - "granularitySpec": { - "type": "uniform", - "segmentGranularity": "DAY", - "queryGranularity": "HOUR", - "rollup": false - } - }, - "tuningConfig": { - "type": "kafka", - "maxRowsPerSegment": 50000, - "logParseExceptions": true - }, - "ioConfig": { - "type": "kafka", - "topic": "obsrv.telemetry.input", - "consumerProperties": {"bootstrap.servers": "localhost:9092"}, - "taskCount": 1, - "replicas": 1, - "taskDuration": "PT8H", - "useEarliestOffset": false, - "completionTimeout": "PT8H", - "inputFormat": { - "type": "json", - "flattenSpec": { - "useFieldDiscovery": true, - "fields": [] - } - }, - "appendToExisting": false - } - } - } - public static INVALID_INGESTION_SPEC = { - "type": "kafka", - "spec": "ingestion_spec" - } -} - -export { TestDruidQuery, TestDataIngestion, TestDataset, TestDataSource, TestDatasetSourceConfig, TestExhaust, TestSubmitIngestion}; \ No newline at end of file diff --git a/api-service/src/v1/test/IngestorTestService.spec.ts b/api-service/src/v1/test/IngestorTestService.spec.ts deleted file mode 100644 index 855cf0c5..00000000 --- a/api-service/src/v1/test/IngestorTestService.spec.ts +++ /dev/null @@ -1,276 +0,0 @@ -import app from "../../app"; -import chai, { expect } from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import httpStatus from "http-status"; -import { TestDataIngestion, TestSubmitIngestion } from "./Fixtures"; -import { config } from "./Config"; -import { routesConfig } from "../configs/RoutesConfig"; -import constants from "../resources/Constants.json" -import { dbConnector, globalCache, ingestorService, kafkaConnector } from "../routes/Router"; -import { describe, it } from 'mocha'; -import nock from "nock"; -import { DatasetStatus } from "../models/DatasetModels"; - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -describe("DATA INGEST API", () => { - afterEach(() => { - chai.spy.restore() - }) - - it("it should ingest data successfully", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" }, "extraction_config": { "is_batch_event": true, "extraction_key": "events", "batch_id": "id" } } ] - }) - chai.spy.on(kafkaConnector.telemetryService, "dispatch", () => { - return Promise.resolve("data ingested") - }) - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - chai.spy.restore(kafkaConnector.telemetryService, "dispatch") - done() - }) - }); - it("it should ingest data successfully for batch even for individual events", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" }, "extraction_config": { "is_batch_event": true, "extraction_key": "events", "batch_id": "id" } } ] - }) - chai.spy.on(kafkaConnector.telemetryService, "dispatch", () => { - return Promise.resolve("data ingested") - }) - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INDIVIDUAL_EVENT) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - chai.spy.restore(kafkaConnector.telemetryService, "dispatch") - done() - }) - }); - it("it should not ingest data successfully when kafka is unable to connect", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" } } ] - }) - chai.spy.on(kafkaConnector.telemetryService, "dispatch", () => { - return Promise.reject("error connecting to kafka") - }) - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - chai.spy.restore(kafkaConnector.telemetryService, "dispatch") - done() - }) - }); - it("it should not ingest data when invalid extraction config present for batch", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": ":datasetId", "status": DatasetStatus.Live, "dataset_config": { "entry_topic": "topic" }, "extraction_config": { "is_batch_event": true, "extraction_key": "eventas", "batch_id": "ids" } } ] - }) - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - chai.spy.restore(kafkaConnector.telemetryService, "dispatch") - done() - }) - }); - it("it should not ingest data successfully", (done) => { - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - chai.spy.on(globalCache, 'get', () => { - return [ { "id": "datasetId", "dataset_config": { "entry_topic": "topic" } } ] - }) - - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "500_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(dbConnector, "listRecords") - chai.spy.restore(globalCache, 'get') - done() - }) - }); - it("it should not ingest data successfully", (done) => { - chai.spy.on(globalCache, 'get', () => { - return [ {} ] - }) - chai.spy.on(dbConnector, "listRecords", () => { - return Promise.resolve([ {} ]) - }) - - chai - .request(app) - .post(config.apiDatasetIngestEndPoint) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "404_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - chai.spy.restore(globalCache, "get") - chai.spy.restore(dbConnector, "listRecords") - done(); - }); - }); - it("it should not ingest data when datasetid param is empty", (done) => { - chai - .request(app) - .post(config.apiDatasetIngestEndPoint.replace(':datasetId', " /")) - .send(TestDataIngestion.SAMPLE_INPUT) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.data_ingest.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done(); - }); - }); - it("it should not establish connection with kafka", (done) => { - chai.spy.on(kafkaConnector.telemetryService, "health", () => { - return Promise.reject("error connecting to kafka") - }) - ingestorService.init() - expect(kafkaConnector.telemetryService.health).to.be.called - chai.spy.restore(kafkaConnector.telemetryService, "health") - done(); - }); - it("it should establish connection with kafka", (done) => { - chai.spy.on(kafkaConnector.telemetryService, "health", () => { - return Promise.resolve("connected to kafka") - }) - ingestorService.init() - expect(kafkaConnector.telemetryService.health).to.be.called - chai.spy.restore(kafkaConnector.telemetryService, "health") - done(); - }); -}) - -describe("SUBMIT INGESTION API", () => { - beforeEach(() => { - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSubmitIngestionEndPoint) - .reply(200) - }) - afterEach(() => { - nock.cleanAll() - }) - it("should submit ingestion successfully", (done)=>{ - chai - .request(app) - .post(config.apiSubmitIngestionEndPoint) - .send(TestSubmitIngestion.VALID_INGESTION_SPEC) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "200_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.submit_ingestion.api_id); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS) - done() - }) - }) - it("should throw error for invalid request object", (done)=>{ - chai - .request(app) - .post(config.apiSubmitIngestionEndPoint) - .send(TestSubmitIngestion.INVALID_INGESTION_SPEC) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus[ "400_NAME" ]); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.submit_ingestion.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done() - }) - }) -}) - -describe("SUBMIT INGESTION ERROR SCENARIOS", ()=>{ - beforeEach(() => { - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSubmitIngestionEndPoint) - .reply(500) - }) - afterEach(() => { - nock.cleanAll() - }) - it("should handle errors", (done)=>{ - chai - .request(app) - .post(config.apiSubmitIngestionEndPoint) - .send(TestSubmitIngestion.VALID_INGESTION_SPEC) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.should.have.property("result"); - res.body.id.should.be.eq(routesConfig.submit_ingestion.api_id); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE) - done() - }) -}) -}) diff --git a/api-service/src/v1/test/QueryTestService.spec.ts b/api-service/src/v1/test/QueryTestService.spec.ts deleted file mode 100644 index bdfb6d60..00000000 --- a/api-service/src/v1/test/QueryTestService.spec.ts +++ /dev/null @@ -1,738 +0,0 @@ -import app from "../../app"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import nock from "nock"; -import httpStatus from "http-status"; -import { TestDruidQuery } from "./Fixtures"; -import { config } from "./Config"; -import constants from "../resources/Constants.json"; -import { routesConfig } from "../configs/RoutesConfig"; -import { dbConnector } from "../routes/Router"; -import { QueryValidator } from "../validators/QueryValidator"; -import chaiSpies from 'chai-spies' -import { describe, it } from 'mocha'; -import * as lakehouseutil from "../helpers/LakehouseUtil"; -chai.use(chaiSpies) -chai.should(); -chai.use(chaiHttp); - -describe("QUERY API", () => { - - describe("If service is down", () => { - - it("it should raise error when native query endpoint is called", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(500) - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - nock.cleanAll(); - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("should raise error when sql query endpoint is called", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(500) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - nock.cleanAll(); - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - }); - describe("POST /query/v2/native-query", () => { - it("it should fetch information from druid data source", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.should.have.property("result"); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject query, when datarange given as list is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_DATE_RANGE_GIVEN_AS_LIST)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject query, when datarange given as string is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_DATE_RANGE_GIVEN_AS_STRING)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should set threshold to default when given threshold is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_THRESHOLD_QUERY)) // given threshold is 1K - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should set row_limit to default when given row_limit is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_LIMIT_QUERY)) // given row_limit is 1K - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should set threshold to default when threshold is not given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.WITHOUT_THRESOLD_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject query when date range is not given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.WITHOUT_DATE_RANGE_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should skip validation and allow druid for query if rules does not exist for datasource", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.UNSUPPORTED_DATA_SOURCE)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should skip validation", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.SKIP_VALIDATION_NATIVE)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject query with unsupported schema", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.UNSUPPORTED_SCHEMA)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - it("it should reject the query because of incorrect url", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .get("/invalid/endpoint") - .end((err, res) => { - res.should.have.status(httpStatus.NOT_FOUND); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.default.api_id); - res.body.responseCode.should.be.eq(httpStatus["404_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }); - }); - }); - describe("POST /druid/v2/sql", () => { - it("it should allow druid to query when a valid sql query is given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should update row_limit to default when row_limit is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_LIMIT_SQL_QUERY)) // given row_limit is 1K - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should set row_limit to default when none is given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.WITHOUT_LIMIT_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); // default is 100 - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should reject the query when daterange range is higher than limit", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.HIGH_DATE_RANGE_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should reject the query when no daterange is given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.WITHOUT_DATE_RANGE_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }); - }); - it("it should skip validation and allow druid for query if rules does not exist for datasource ", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.UNSUPPORTED_DATASOURCE_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }) - }) - it("it should skip validation", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [ { "datasource_ref": "sample_ref" } ] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [ { events: [] } ]); - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.SKIP_VALIDATION_SQL)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.SUCCESS); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - nock.cleanAll() - done(); - }) - }) - it("should throw error for invalid sql query", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, ["sample_ref"]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [{ events: [] }]); - chai. - request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.INVALID_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords"); - nock.cleanAll(); - done(); - }) - }) - it("should throw error is table name is missing from the SQL Query", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, ["sample_ref"]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [{ events: [] }]); - chai. - request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.MISSING_TABLE_NAME)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords"); - nock.cleanAll(); - done(); - }) - }) - }) - describe("error scenarios", () => { - it("should handle the error", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - throw new Error("error occured while fetching records") - }) - nock(config.druidHost + ":" + config.druidPort) - .get(config.druidDatasourcesEndPoint) - .reply(200, [ "sample_ref" ]) - nock(config.druidHost + ":" + config.druidPort) - .post(config.druidSqlEndPoint) - .reply(200, [{ events: [] }]); - - chai - .request(app) - .post(config.apiDruidEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.query.native_query.api_id); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - nock.cleanAll() - chai.spy.restore(dbConnector, "readRecords") - done(); - }) - }) - }) - it("should not validate if called with invalid url", async () => { - const queryValidator = new QueryValidator() - await queryValidator.validate({}, "obsrv.api") - .then((result) => { - result.isValid.should.be.equal(false) - }) - }), - describe("Datalake query API POST /data/v1/sql-query", () => { - - it("it should retrieve data from datalake when a valid sql query is given", (done) => { - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - chai.spy.on(lakehouseutil, "executeLakehouseQuery", () => { - return Promise.resolve([{ totalRatingsCount: 10 }]) - }) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY_DATALAKE)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - chai.spy.restore(lakehouseutil, "executeLakehouseQuery") - done(); - }); - }) - it("it should handle error scenarios in datalake query", (done)=>{ - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - chai.spy.on(lakehouseutil, "executeLakehouseQuery", () => { - throw new Error("error occured while fetching records") - }) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY_DATALAKE)) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - res.body.responseCode.should.be.eq(httpStatus["500_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - chai.spy.restore(dbConnector, "readRecords") - chai.spy.restore(lakehouseutil, "executeLakehouseQuery") - done(); - }) - }) - it("should throw error incase of invalid datasource type", (done)=>{ - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.INVALID_DATASOURCE_TYPE)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - done(); - }) - }) - it("should throw error incase of empty sql query", (done)=>{ - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.EMPTY_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); - res.body.should.be.a("object"); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - res.body.responseCode.should.be.eq(httpStatus["400_NAME"]); - res.body.params.status.should.be.eq(constants.STATUS.FAILURE); - res.body.result.should.be.empty; - - done(); - }) - }) - it("should handle native scenarios in case of trino query", (done)=>{ - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - chai.spy.on(lakehouseutil, "executeLakehouseQuery", () => { - return Promise.resolve([{ totalRatingsCount: 10 }]) - }) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.VALID_QUERY_DATALAKE)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - chai.spy.restore(lakehouseutil, "executeLakehouseQuery") - done(); - }); - }) - it("should consider sql query as case insensitive", (done)=>{ - chai.spy.on(dbConnector, "readRecords", () => { - return [{ "datasource_ref": "sample_ref" }] - }) - chai.spy.on(lakehouseutil, "executeLakehouseQuery", () => { - return Promise.resolve([{ totalRatingsCount: 10 }]) - }) - chai - .request(app) - .post(config.apiDruidSqlEndPoint) - .send(JSON.parse(TestDruidQuery.CASE_INSENSITIVE_SQL_QUERY)) - .end((err, res) => { - res.should.have.status(httpStatus.OK); - res.body.should.be.a("object"); - res.body.responseCode.should.be.eq(httpStatus["200_NAME"]); - res.body.result.length.should.be.lessThan(101); - res.body.id.should.be.eq(routesConfig.query.sql_query.api_id); - chai.spy.restore(dbConnector, "readRecords") - chai.spy.restore(lakehouseutil, "executeLakehouseQuery") - done(); - }); - }) - }) -}) \ No newline at end of file diff --git a/api-service/src/v1/test/TestConnector.spec.ts b/api-service/src/v1/test/TestConnector.spec.ts deleted file mode 100644 index 64cdaf21..00000000 --- a/api-service/src/v1/test/TestConnector.spec.ts +++ /dev/null @@ -1,193 +0,0 @@ -import { DbConnector } from "../connectors/DbConnector"; -import { KafkaConnector } from "../connectors/KafkaConnector"; -import { HTTPConnector } from "../connectors/HttpConnector"; -import chai from "chai"; -import chaiHttp from "chai-http"; -import spies from "chai-spies"; -import { describe, it } from 'mocha'; - - -chai.use(spies); -chai.should(); -chai.use(chaiHttp); - -const dbConnectorConfig = { - client: "postgresql", - connection: { - host: 'localhost', - port: 5432, - database: 'test', - user: 'test', - password: 'test', - } -} - -describe("Testing Db connector", () => { - it("should successfully arrange connection with database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "select", () => { - return Promise.resolve() - }) - dbConnector.init() - chai.expect(dbConnector.pool.select).to.be.called - chai.spy.restore(dbConnector.pool, "select") - done() - }) - it("should not connect to database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "connect", () => { - return Promise.reject("error occurred while connecting to postgres") - }) - dbConnector.init() - chai.expect(dbConnector.connect).to.be.called - chai.spy.restore(dbConnector, "connect") - done() - }) - it("should not insert records into database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "insertRecord", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - dbConnector.execute("insert", { "table": "users", "fields": {} }) - chai.expect(dbConnector.insertRecord).to.be.called - chai.spy.restore(dbConnector, "insertRecord") - done() - }) - it("should insert record into database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "insertRecord", () => { - return Promise.resolve([]) - }) - dbConnector.execute("insert", { "table": "users", "fields": {} }) - chai.expect(dbConnector.insertRecord).to.be.called - chai.spy.restore(dbConnector, "insertRecord") - done() - }) - it("should not update records in database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "updateRecord", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - dbConnector.execute("update", { "table": "users", "fields": {} }) - chai.expect(dbConnector.updateRecord).to.be.called - chai.spy.restore(dbConnector, "updateRecord") - done() - }) - it("should update records in database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "updateRecord", () => { - return Promise.resolve([]) - }) - dbConnector.execute("update", { "table": "users", "fields": {} }) - chai.expect(dbConnector.updateRecord).to.be.called - chai.spy.restore(dbConnector, "updateRecord") - done() - }) - it("should throw error while updating records into database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "transaction", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - dbConnector.execute("update", { "table": "users", "fields": {} }) - chai.expect(dbConnector.updateRecord).throw - chai.spy.restore(dbConnector.pool, "transaction") - done() - }) - it("should not throw error while updating records into database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "transaction", () => { - return (Promise.resolve([])) - }) - dbConnector.execute("update", { "table": "users", "fields": {} }) - chai.expect(dbConnector.updateRecord).to.not.throw - chai.spy.restore(dbConnector.pool, "transaction") - done() - }) - it("should not retrieve records from database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "readRecords", () => { - return Promise.reject(new Error("error occurred while connecting to postgres")) - }) - dbConnector.execute("read", { "table": "users", "fields": {} }) - chai.expect(dbConnector.readRecords).to.be.called - chai.spy.restore(dbConnector, "readRecords") - done() - }) - it("should retrieve records from database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector, "readRecords", () => { - return Promise.resolve([]) - }) - dbConnector.execute("read", { "table": "users", "fields": {} }) - chai.expect(dbConnector.readRecords).to.be.called - chai.spy.restore(dbConnector, "readRecords") - done() - }) - it("should not disconnect from the database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "destroy", () => { - return Promise.reject(new Error("error occurred while disconnecting from postgres")) - }) - dbConnector.close() - chai.expect(dbConnector.pool.destroy).to.throw - chai.spy.restore(dbConnector.pool, "destroy") - done() - }) - it("should disconnect from the database", (done) => { - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "destroy", () => { - return Promise.resolve() - }) - dbConnector.close() - chai.expect(dbConnector.pool.destroy).to.not.throw - chai.spy.restore(dbConnector.pool, "destroy") - done() - }) - it("should list records", (done)=>{ - const dbConnector = new DbConnector(dbConnectorConfig); - chai.spy.on(dbConnector.pool, "select", () => { - return Promise.resolve() - }) - dbConnector.listRecords('datasets') - chai.expect(dbConnector.pool.select).to.not.throw - chai.spy.restore(dbConnector.pool, "select") - done() - }) -}) -describe('testing httpConnector', () => { - describe('execute', () => { - it('should throw an error', (done) => { - const sampleHttpInstance = new HTTPConnector('example.com'); - const spy = chai.spy.on(sampleHttpInstance, 'execute'); - (() => sampleHttpInstance.execute('sample string')).should.throw(Error); - spy.should.have.been.called.once; - chai.spy.restore(sampleHttpInstance, 'execute') - done() - }); - }); - - describe('close', () => { - it('should throw an error', (done) => { - const sampleHttpInstance = new HTTPConnector('example.com'); - const spy = chai.spy.on(sampleHttpInstance, 'close'); - (() => sampleHttpInstance.close()).should.throw(Error); - spy.should.have.been.called.once; - chai.spy.restore(sampleHttpInstance, 'execute') - done() - }); - }); - -}) -describe("testing kafkaConnector", () => { - describe('close', () => { - it('should throw an error', (done) => { - const sampleKafkaInstance = new KafkaConnector(); - const spy = chai.spy.on(sampleKafkaInstance, 'close'); - (() => sampleKafkaInstance.close()).should.throw(Error); - spy.should.have.been.called.once; - chai.spy.restore(sampleKafkaInstance, 'execute') - done() - }); - }); - -}) \ No newline at end of file diff --git a/api-service/src/v1/utils/common.ts b/api-service/src/v1/utils/common.ts deleted file mode 100644 index eed6006d..00000000 --- a/api-service/src/v1/utils/common.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { DateRange } from "../models/ExhaustModels"; -import moment from "moment"; -import * as _ from "lodash"; -import { Request, Response } from "express"; - -export const getPeriodInterval = (since: String): DateRange => { - const period = since.toLowerCase().split("_")[1]; - return { - from: moment().subtract(period, "days").format("yyyy-MM-DD"), - to: moment().startOf("day").format("yyyy-MM-DD"), - }; -}; - -export const getDateRange = (request: Request, response: Response) => { - const { from, to, since } = request.query; - let period; - if (!_.isUndefined(since)) { - period = getPeriodInterval(since.toString()); - } else if (from != undefined && to != undefined) { - period = { from: from.toString(), to: to.toString() }; - } else { - period = getPeriodInterval("LAST_0_DAYS"); - } - return period; -}; - -export const getFileKey = (key: string): string => { - let keyArray: string[] = key.split("/"); - return keyArray[keyArray.length - 1].slice(0, 10); -}; - -export const isValidDateRange = ( - fromDate: moment.Moment, toDate: moment.Moment, allowedRange: number = 0 -): boolean => { - const differenceInDays = Math.abs(fromDate.diff(toDate, "days")); - const isValidDates = differenceInDays > allowedRange ? false : true; - return isValidDates; -}; diff --git a/api-service/src/v1/validators/QueryValidator.ts b/api-service/src/v1/validators/QueryValidator.ts deleted file mode 100644 index 74978227..00000000 --- a/api-service/src/v1/validators/QueryValidator.ts +++ /dev/null @@ -1,228 +0,0 @@ -import httpStatus from "http-status"; -import _ from "lodash"; -import moment, { Moment } from "moment"; -import { queryRules } from "../configs/QueryRules"; -import { IConnector, IValidator } from "../models/DatasetModels"; -import { ICommonRules, ILimits, IQuery, IQueryTypeRules, IRules } from "../models/QueryModels"; -import { ValidationStatus } from "../models/ValidationModels"; -import constants from "../resources/Constants.json"; -import { dbConnector } from "../routes/Router"; -import { routesConfig } from "../configs/RoutesConfig"; -import { config } from "../configs/Config"; -import { isValidDateRange } from "../utils/common"; -import { HTTPConnector } from "../connectors/HttpConnector"; -export class QueryValidator implements IValidator { - private limits: ILimits; - private momentFormat: string; - private httpConnector: any - constructor() { - this.limits = queryRules - this.momentFormat = "YYYY-MM-DD HH:MI:SS" - this.httpConnector = new HTTPConnector(`${config.query_api.druid.host}:${config.query_api.druid.port}`).connect() - } - public async validate(data: any, id: string): Promise { - let validationStatus, dataSource, shouldSkip; - switch (id) { - case routesConfig.query.native_query.api_id: - validationStatus = await this.validateNativeQuery(data) - dataSource = this.getDataSource(data) - shouldSkip = _.includes(config.exclude_datasource_validation, dataSource); - return validationStatus.isValid ? (shouldSkip ? validationStatus : this.setDatasourceRef(dataSource, data)) : validationStatus - case routesConfig.query.sql_query.api_id: - validationStatus = await this.validateSqlQuery(data) - if (validationStatus.isValid) { - dataSource = this.getDataSource(data) - shouldSkip = _.includes(config.exclude_datasource_validation, dataSource); - return validationStatus.isValid ? (shouldSkip ? validationStatus : this.setDatasourceRef(dataSource, data)) : validationStatus - } - return validationStatus - default: - return { isValid: false } - } - } - - private validateNativeQuery(data: any): ValidationStatus { - let queryObj: IQuery = data; - this.setQueryLimits(data, this.limits.common); - let dataSourceLimits = this.getDataSourceLimits(this.getDataSource(data)); - try { - return (!_.isEmpty(dataSourceLimits)) ? this.validateQueryRules(queryObj, dataSourceLimits.queryRules[queryObj.query.queryType as keyof IQueryTypeRules]) : { isValid: true } - } catch (error: any) { - return { isValid: false, message: error.message || "error ocuured while validating native query", code: error.code || httpStatus["400_NAME"] }; - } - } - - private validateSqlQuery(data: IQuery): ValidationStatus { - try { - let query = data.querySql.query; - if (_.isEmpty(query)) { - return { isValid: false, message: "Query must not be empty", code: httpStatus["400_NAME"] }; - } - const fromClause = /\bFROM\b/i; - const isFromClausePresent = fromClause.test(query) - if (!isFromClausePresent) { - return { isValid: false, message: "Invalid SQL Query", code: httpStatus["400_NAME"] }; - } - const fromIndex = query.search(fromClause); - const dataset = query.substring(fromIndex + 4).trim().split(/\s+/)[0].replace(/\\/g, ""); - if (_.isEmpty(dataset)) { - return { isValid: false, message: "Dataset name must be present in the SQL Query", code: httpStatus["400_NAME"] }; - } - this.setQueryLimits(data, this.limits.common); - let datasource = this.getDataSource(data); - let dataSourceLimits = this.getDataSourceLimits(datasource); - return (!_.isEmpty(dataSourceLimits)) ? this.validateQueryRules(data, dataSourceLimits.queryRules.scan) : { isValid: true }; - } catch (error: any) { - return { isValid: false, message: error.message || "error ocuured while validating SQL query", code: error.code || httpStatus["500_NAME"] }; - } - } - - private validateQueryRules(queryPayload: IQuery, limits: IRules): ValidationStatus { - let fromDate: Moment | undefined, toDate: Moment | undefined; - let allowedRange = limits.maxDateRange; - if (queryPayload.query) { - const dateRange = this.getIntervals(queryPayload.query); - const extractedDateRange = Array.isArray(dateRange) ? dateRange[0].split("/") : dateRange.toString().split("/"); - fromDate = moment(extractedDateRange[0], this.momentFormat); - toDate = moment(extractedDateRange[1], this.momentFormat); - } else { - let query = queryPayload.querySql.query; - query = query.toUpperCase().replace(/\s+/g, " ").trim(); - let vocabulary = query.split(/\s+/); - let fromDateIndex = vocabulary.indexOf("TIMESTAMP"); - let toDateIndex = vocabulary.lastIndexOf("TIMESTAMP"); - fromDate = moment(vocabulary[fromDateIndex + 1], this.momentFormat); - toDate = moment(vocabulary[toDateIndex + 1], this.momentFormat); - } - const isValidDates = fromDate && toDate && fromDate.isValid() && toDate.isValid() - return isValidDates ? this.validateDateRange(fromDate, toDate, allowedRange) - : { isValid: false, message: constants.ERROR_MESSAGE.NO_DATE_RANGE, code: httpStatus["400_NAME"] }; - }; - - private getDataSource(queryPayload: IQuery): string { - if (queryPayload.querySql) { - let query = queryPayload.querySql.query; - query = query.replace(/\s+/g, " ").trim(); - const fromIndex = query.search(/\bFROM\b/i); - const dataSource = query.substring(fromIndex).split(/\s+/)[1].replace(/\\/g, "").replace(/"/g, ""); - return dataSource; - } else { - const dataSourceField: any = queryPayload.query.dataSource - if (typeof dataSourceField == 'object') { return dataSourceField.name } - return dataSourceField - } - }; - - private getDataSourceLimits(datasource: string): any { - for (var index = 0; index < this.limits.rules.length; index++) { - if (this.limits.rules[index].dataset == datasource) { - return this.limits.rules[index]; - } - } - }; - - private validateDateRange(fromDate: moment.Moment, toDate: moment.Moment, allowedRange: number = 0): ValidationStatus { - const isValidDates = isValidDateRange(fromDate, toDate, allowedRange); - return isValidDates - ? { isValid: true, code: httpStatus[200] } - : { isValid: false, message: constants.ERROR_MESSAGE.INVALID_DATE_RANGE.replace("${allowedRange}", allowedRange.toString()), code: httpStatus["400_NAME"] }; - }; - - private getLimit(queryLimit: number, maxRowLimit: number) { - return queryLimit > maxRowLimit ? maxRowLimit : queryLimit; - }; - - private setQueryLimits(queryPayload: IQuery, limits: ICommonRules) { - if (queryPayload.query) { - if (queryPayload.query.threshold) { - queryPayload.query.threshold = this.getLimit(queryPayload.query.threshold, limits.maxResultThreshold); - } else { - queryPayload.query.threshold = limits.maxResultThreshold; - } - if (queryPayload.query.limit) { - queryPayload.query.limit = this.getLimit(queryPayload.query.limit, limits.maxResultRowLimit); - } else { - queryPayload.query.limit = limits.maxResultRowLimit; - } - } else { - const limitClause = /\bLIMIT\b/i; - const vocabulary = queryPayload.querySql.query.split(/\s+/); // Splitting the query by whitespace - const queryLimitIndex = vocabulary.findIndex(word => limitClause.test(word)); - const queryLimit = Number(vocabulary[queryLimitIndex + 1]); - - if (isNaN(queryLimit)) { - // If "LIMIT" clause doesn't exist or its value is not a number, update the query - const updatedVocabulary = [...vocabulary, "LIMIT", limits.maxResultRowLimit]; - queryPayload.querySql.query = updatedVocabulary.join(" "); - } else { - // If "LIMIT" clause exists and its value is a number, update the limit - const newLimit = this.getLimit(queryLimit, limits.maxResultRowLimit); - vocabulary[queryLimitIndex + 1] = newLimit.toString(); - queryPayload.querySql.query = vocabulary.join(" "); - } - } - } - public async validateDatasource(datasource: any) { - let existingDatasources = await this.httpConnector(config.query_api.druid.list_datasources_path, {}) - if (!_.includes(existingDatasources.data, datasource)) { - let error = constants.INVALID_DATASOURCE - error.message = error.message.replace('${datasource}', datasource) - throw error - } - return - } - public async setDatasourceRef(dataSource: string, payload: any): Promise { - try { - const granularity = _.get(payload, 'context.granularity') - const dataSourceType = _.get(payload, 'context.dataSourceType', config.query_api.druid.queryType) - let dataSourceRef = await this.getDataSourceRef(dataSource, granularity, dataSourceType); - if (dataSourceType === config.query_api.druid.queryType) await this.validateDatasource(dataSourceRef) - if (payload?.querySql && dataSourceType === config.query_api.druid.queryType) { - payload.querySql.query = payload.querySql.query.replace(dataSource, dataSourceRef) - } - else if (payload?.querySql && dataSourceType === config.query_api.lakehouse.queryType) { - // hudi tables doesn't support table names contain '-' so we need to replace it with '_' - let modifiedDataSource = dataSourceRef.replace(/-/g, "_") - payload.querySql.query = payload.querySql.query.replace(dataSource, modifiedDataSource).replace(/"/g, "") - } - else { - payload.query.dataSource = dataSourceRef - } - return { isValid: true }; - } catch (error: any) { - return { isValid: false, message: error.message || "error ocuured while fetching datasource record", code: error.code || httpStatus["400_NAME"] }; - } - } - - public async getDataSourceRef(datasource: string, granularity: string | undefined, dataSourceType: string): Promise { - let storageType = dataSourceType === config.query_api.lakehouse.queryType ? config.datasource_storage_types.datalake : config.datasource_storage_types.druid - const records: any = await dbConnector.readRecords("datasources", { "filters": { "dataset_id": datasource, "type": storageType } }) - if (records.length == 0) { - const error = { ...constants.INVALID_DATASOURCE } - error.message = error.message.replace('${datasource}', datasource) - throw error - } - if (storageType === config.datasource_storage_types.datalake) return `${config.query_api.lakehouse.catalog}.${config.query_api.lakehouse.schema}.${records[0].datasource_ref}_ro` - const record = records.filter((record: any) => { - const aggregatedRecord = _.get(record, "metadata.aggregated") - if (granularity) - return aggregatedRecord && _.get(record, "metadata.granularity") === granularity; - else - return !aggregatedRecord - }); - - if (record.length == 0) { - const error = { ...constants.INVALID_DATASOURCE } - error.message = error.message.replace('${datasource}', datasource) - if (granularity !== undefined) { - error.message = `Aggregate ${error.message}` - } - throw error - } - return record[0].datasource_ref - } - - private getIntervals(payload: any) { - return (typeof payload.intervals == 'object' && !Array.isArray(payload.intervals)) ? payload.intervals.intervals : payload.intervals - } -} diff --git a/api-service/src/v1/validators/RequestsValidator.ts b/api-service/src/v1/validators/RequestsValidator.ts deleted file mode 100644 index 871c59ad..00000000 --- a/api-service/src/v1/validators/RequestsValidator.ts +++ /dev/null @@ -1,87 +0,0 @@ -import Ajv from "ajv"; -import addFormats from "ajv-formats"; -import fs from "fs"; -import httpStatus from "http-status"; -import { IValidator } from "../models/DatasetModels"; -import { ValidationStatus } from "../models/ValidationModels"; -import { routesConfig } from "../configs/RoutesConfig"; - -export class RequestsValidator implements IValidator { - private schemaBasePath: string = "/src/v1/resources/"; - private reqSchemaMap = new Map(); - private validator: Ajv; - - constructor() { - this.validator = new Ajv(); - addFormats(this.validator); - this.loadSchemas(); - } - - validate(data: any, id: string): ValidationStatus { - return this.validateRequest(data, id); - } - - validateQueryParams(data: any, id: string): ValidationStatus { - return this.validateRequestParams(data, id); - } - - private validateRequest(data: any, id: string): ValidationStatus { - let validRequestObj = this.validator.validate(this.getReqSchema(id), data); - if (!validRequestObj) { - let error = this.validator.errors; - let errorMessage = error![0].instancePath.replace("/", "") + " " + error![0].message; - return { error: httpStatus["400_NAME"], isValid: false, message: errorMessage, code: httpStatus["400_NAME"] }; - } else { - return { isValid: true, message: "Validation Success", code: httpStatus[200] }; - } - }; - - private validateRequestParams(data: any, id: string): ValidationStatus { - let validRequestObj = this.validator.validate(this.getReqSchema(id), data); - if (!validRequestObj) { - let error = this.validator.errors; - const property = error![0].instancePath.replace("/", ""); - let errorMessage = `property \"${property}\"` + " " + error![0].message; - return { error: httpStatus["400_NAME"], isValid: false, message: errorMessage, code: httpStatus["400_NAME"] }; - } else { - return { isValid: true, message: "Validation Success", code: httpStatus[200] }; - } - }; - - private schemasMapping(): Record { - return [ - routesConfig.query.native_query, - routesConfig.query.sql_query, - routesConfig.data_ingest, - routesConfig.tenant_ingest, - routesConfig.config.dataset.save, - routesConfig.config.datasource.save, - routesConfig.config.dataset.list, - routesConfig.config.datasource.list, - routesConfig.config.dataset.update, - routesConfig.config.datasource.update, - routesConfig.config.dataset_source_config.save, - routesConfig.config.dataset_source_config.update, - routesConfig.config.dataset_source_config.list, - routesConfig.exhaust, - routesConfig.submit_ingestion, - routesConfig.schema_validator - ] - } - - private loadSchemas(): void { - this.schemasMapping().map((routeObject: Record) => - this.reqSchemaMap.set(routeObject.api_id, this.loadSchemaFromFile(routeObject.validation_schema)) - ) - } - - private loadSchemaFromFile(filename: string): any { - const filePath = process.cwd() + `${this.schemaBasePath}schemas/` + filename; - const fileContent = fs.readFileSync(filePath, "utf8"); - return JSON.parse(fileContent); - } - - private getReqSchema(apiId: string): any { - return this.reqSchemaMap.get(apiId); - } -} diff --git a/api-service/src/v2/configs/QueryRules.ts b/api-service/src/v2/configs/QueryRules.ts deleted file mode 100644 index be5709cc..00000000 --- a/api-service/src/v2/configs/QueryRules.ts +++ /dev/null @@ -1,56 +0,0 @@ -const maxDateRange = process.env.max_date_range ? parseInt(process.env.max_date_range) : 30 // in days - -export const queryRules = { - "common": { - "maxResultThreshold": process.env.max_query_threshold ? parseInt(process.env.max_query_threshold) : 5000, - "maxResultRowLimit": process.env.max_query_limit ? parseInt(process.env.max_query_limit) : 5000 - }, - "rules": [ - { - "dataset": "telemetry-events", - "queryRules": { - "groupBy": { - "maxDateRange": maxDateRange - }, - "scan": { - "maxDateRange": maxDateRange - }, - "search": { - "maxDateRange": maxDateRange - }, - "timeBoundary": { - "maxDateRange": maxDateRange - }, - "timeseries": { - "maxDateRange": maxDateRange - }, - "topN": { - "maxDateRange": maxDateRange - } - } - }, - { - "dataset": "summary-events", - "queryRules": { - "groupBy": { - "maxDateRange": maxDateRange - }, - "scan": { - "maxDateRange": maxDateRange - }, - "search": { - "maxDateRange": maxDateRange - }, - "timeBoundary": { - "maxDateRange": maxDateRange - }, - "timeseries": { - "maxDateRange": maxDateRange - }, - "topN": { - "maxDateRange": maxDateRange - } - } - } - ] -} diff --git a/api-service/src/v2/services/HealthService.ts b/api-service/src/v2/services/HealthService.ts deleted file mode 100644 index c8eca736..00000000 --- a/api-service/src/v2/services/HealthService.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { AxiosInstance } from "axios"; -import { NextFunction, Request, Response } from "express"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; -import { IConnector } from "../types/DatasetModels"; -// import { DbConnector } from "../connections/databaseConnection"; -// import { KafkaConnector } from "../connections/kafkaConnection"; - - - - -export class HealthService { - // private dbConnector: DbConnector; - // private kafkaConnector: KafkaConnector; - private httpDruidConnector: AxiosInstance; - private errorHandler: ErrorResponseHandler; - constructor(dbConnector: any, kafkaConnector:any, httpDruidConnector: IConnector) { - this.errorHandler = new ErrorResponseHandler("HealthService"); - this.httpDruidConnector = httpDruidConnector.connect() - // this.dbConnector = dbConnector; - // this.kafkaConnector = kafkaConnector; - } - - checkHealth(req: Request, res: Response, next: NextFunction) { - Promise.all([this.checkDruidHealth(), this.checkKafkaHealth(), this.checkPostgresHealth()]) - .then(() => { - ResponseHandler.successResponse(req, res, { status: 200, data: {} }) - }).catch(error => { - this.errorHandler.handleError(req, res, next, error) - }) - } - - private async checkDruidHealth() { - await this.httpDruidConnector.get("/status/health") - } - - private async checkKafkaHealth() { - // await this.kafkaConnector.connect() - } - - private async checkPostgresHealth() { - // await this.dbConnector.health() - } - -} diff --git a/api-service/src/v2/telemetry/telemetryActions.ts b/api-service/src/v2/telemetry/telemetryActions.ts deleted file mode 100644 index bbee6faa..00000000 --- a/api-service/src/v2/telemetry/telemetryActions.ts +++ /dev/null @@ -1,21 +0,0 @@ -export default { - "createDataset": "dataset:create", - "updateDataset": "dataset:update", - "readDataset": "dataset:read", - "listDatasets": "dataset:list", - "createDatasource": "datasource:create", - "updateDatasource": "datasource:update", - "getDatasource": "datasource:get", - "listDatasource": "datasource:list", - "createTransformation": "transformation:create", - "updateTransformation": "transformation:update", - "createDatasetSourceConfig": "datasetSourceConfig:create", - "updateDatasetSourceConfig": "datasetSourceConfig:update", - "getatasetSourceConfig": "datasetSourceConfig:get", - "listDatasetSourceConfig": "datasetSourceConfig:list", - "nativeQuery": "dataset:query:native", - "sqlQuery": "dataset:query:sql", - "ingestEvents": "dataset:events:ingest", - "submitIngestionSpec": "datasource:ingestion:submit", - "datasetExhaust": "dataset:exhaust:get" -} \ No newline at end of file diff --git a/api-service/src/v2/types/ConnectionModels.ts b/api-service/src/v2/types/ConnectionModels.ts deleted file mode 100644 index 02a4d700..00000000 --- a/api-service/src/v2/types/ConnectionModels.ts +++ /dev/null @@ -1,13 +0,0 @@ - -export interface DbConfig { - host: string; - port: string | number; - database: string; - user: string; - password: string; -} - -export interface DbConnectorConfig { - client: string; - connection: DbConfig; -} diff --git a/api-service/src/v2/types/ExhaustModels.ts b/api-service/src/v2/types/ExhaustModels.ts deleted file mode 100644 index 2c2acf4b..00000000 --- a/api-service/src/v2/types/ExhaustModels.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface DateRange { - from: string, - to: string -} diff --git a/api-service/src/v2/types/QueryModels.ts b/api-service/src/v2/types/QueryModels.ts deleted file mode 100644 index 6e5f414b..00000000 --- a/api-service/src/v2/types/QueryModels.ts +++ /dev/null @@ -1,66 +0,0 @@ -export interface IDataSourceRules { - dataset: string; - queryRules: IQueryTypeRules; -} - -export interface ICommonRules { - maxResultThreshold: number; - maxResultRowLimit: number; -} - -export interface IQueryTypeRules { - groupBy: IRules; - scan: IRules; - topN: IRules; - timeseries: IRules; - timeBoundary: IRules; - search: IRules; -} - -export interface IRules { - maxDateRange?: number; -} - -export interface IFilter { - type?: string; - fields?: IFilter[]; - field?: IFilter; - dimension?: string; - dimensions?: string[]; -} - -interface ISqlQueryObject { - query: string; -} - -interface ISqlQuery { - context: object; - querySql: ISqlQueryObject; - query: never; -} - -interface INativeQueryObj { - queryType: string; - dataSource: string; - dimension?: string; - dimensions?: string[]; - filter?: IFilter; - aggregations?: any[]; - postAggregations?: any[]; - granularity: string; - limit?: number; - threshold?: number; - intervals: string[] | string; -} - -interface INativeQuery { - context: object; - query: INativeQueryObj; - querySql: never; -} - -export interface ILimits { - common: ICommonRules; - rules: IDataSourceRules[]; -} -export type IQuery = ISqlQuery | INativeQuery; diff --git a/api-service/src/v2/types/ValidationModels.ts b/api-service/src/v2/types/ValidationModels.ts deleted file mode 100644 index 993e3962..00000000 --- a/api-service/src/v2/types/ValidationModels.ts +++ /dev/null @@ -1,6 +0,0 @@ -export interface ValidationStatus { - isValid: boolean, - message?: string, - error?: string, - code?: string -} \ No newline at end of file diff --git a/api-service/src/v2/validators/RequestsValidator.ts b/api-service/src/validators/RequestsValidator.ts similarity index 100% rename from api-service/src/v2/validators/RequestsValidator.ts rename to api-service/src/validators/RequestsValidator.ts From 449ed9a43d0c9ef9686fc8071c2d52de6c73d8bf Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 24 Jul 2024 22:12:26 +0530 Subject: [PATCH 055/235] #OBS-I1 Added DatasethealthService --- .../src/v2/services/DatasetHealthService.ts | 485 ++++++++++++++++++ 1 file changed, 485 insertions(+) create mode 100644 api-service/src/v2/services/DatasetHealthService.ts diff --git a/api-service/src/v2/services/DatasetHealthService.ts b/api-service/src/v2/services/DatasetHealthService.ts new file mode 100644 index 00000000..270496fe --- /dev/null +++ b/api-service/src/v2/services/DatasetHealthService.ts @@ -0,0 +1,485 @@ +import axios from "axios"; +import { config } from "../configs/Config"; +import { logger } from "@project-sunbird/logger"; +import { health as postgresHealth } from "../connections/databaseConnection"; +import { DatasetType, DataSourceType, HealthStatus } from "../types/DatasetModels"; +import { createClient } from "redis"; +import { isHealthy as isKafkaHealthy } from "../connections/kafkaConnection"; +import { druidHttpService, executeNativeQuery } from "../connections/druidConnection"; +import _ from "lodash"; +import moment from "moment"; +import { SystemConfig } from "./SystemConfig"; +import { datasetService } from "./DatasetService"; +const dateFormat = "YYYY-MM-DDT00:00:00+05:30" + +const prometheusInstance = axios.create({ baseURL: config?.query_api?.prometheus?.url, headers: { "Content-Type": "application/json" } }) + +const prometheusQueries = { + validationFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_DATASETID_validator_failed_count[1d]))", + dedupFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_DATASETID_dedup_failed_count[1d]))", + denormFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_DenormalizerJob_DATASETID_denorm_failed[1d]))", + transformationFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_TransformerJob_DATASETID_transform_failed_count[1d]))", + queriesCount: 'sum(sum_over_time(node_total_api_calls{entity="data-out", dataset_id="DATASETID"}[1d]))', + avgQueryResponseTimeInSec: 'avg(avg_over_time(node_query_response_time{entity="data-out", dataset_id="DATASETID"}[1d]))/1000', + queriesFailedCount: 'sum(sum_over_time(node_failed_api_calls{entity="data-out", dataset_id="DATASETID"}[1d]))' +} + +export const getDatasetHealth = async (categories: any, dataset: any) => { + + const details = [] + if (categories.includes("infra")) { + const isMasterDataset = _.get(dataset, "type") == DatasetType.master; + const { components, status } = await getInfraHealth(isMasterDataset) + details.push({ + "category": "infra", + "status": status, + "components": components + }) + } + if (categories.includes("processing")) { + const { components, status } = await getProcessingHealth(dataset) + details.push({ + "category": "processing", + "status": status, + "components": components + }) + } + + if (categories.includes("query")) { + const datasources = await datasetService.findDatasources({ dataset_id: dataset.id, type: DataSourceType.druid }, ["dataset_id", "datasource"]) + const { components, status } = await getQueryHealth(datasources, dataset) + details.push({ + "category": "query", + "status": status, + "components": components + }) + } + + const allStatus = _.includes(_.map(details, (detail) => detail?.status), HealthStatus.UnHealthy) ? HealthStatus.UnHealthy : HealthStatus.Healthy + return { + "status": allStatus, + "details": details + } +} +const getDatasetIdForMetrics = (datasetId: string) => { + datasetId = datasetId.replace(/-/g, "_") + .replace(/\./g, "_") + .replace(/\n/g, "") + .replace(/[\n\r]/g, "") + return datasetId; +} + +const queryMetrics = async (datasetId: string, query: string) => { + const queryWithDatasetId = query.replace("DATASETID", getDatasetIdForMetrics(datasetId)) + try { + const { data } = await prometheusInstance.get("/api/v1/query", { params: {query: queryWithDatasetId} }) + return { count: _.toInteger(_.get(data, "data.result[0].value[1]", "0")) || 0, health: HealthStatus.Healthy } + } catch (error) { + logger.error(error) + return { count: 0, health: HealthStatus.UnHealthy } + } + +} + +export const getInfraHealth = async (isMasterDataset: boolean): Promise<{ components: any, status: string }> => { + const postgres = await getPostgresStatus() + const druid = await getDruidHealthStatus() + const flink = await getFlinkHealthStaus() + const kafka = await getKafkaHealthStatus() + let redis = HealthStatus.Healthy + const components = [ + { "type": "postgres", "status": postgres }, + { "type": "kafka", "status": kafka }, + { "type": "druid", "status": druid }, + { "type": "flink", "status": flink } + ] + if (isMasterDataset) { + redis = await getRedisHealthStatus() + components.push({ "type": "redis", "status": redis }) + } + const status = [postgres, redis, kafka, druid, flink].includes(HealthStatus.UnHealthy) ? HealthStatus.UnHealthy : HealthStatus.Healthy + return { components, status }; +} + +export const getProcessingHealth = async (dataset: any): Promise<{ components: any, status: string }> => { + const dataset_id = _.get(dataset, "dataset_id") + const isMasterDataset = _.get(dataset, "type") == DatasetType.master; + const flink = await getFlinkHealthStaus() + const { count, health } = await getEventsProcessedToday(dataset_id, isMasterDataset) + const processingDefaultThreshold = await SystemConfig.getThresholds("processing") + // eslint-disable-next-line prefer-const + let { count: avgCount, health: avgHealth } = await getAvgProcessingSpeedInSec(dataset_id, isMasterDataset) + if (avgHealth == HealthStatus.Healthy) { + if (avgCount > processingDefaultThreshold?.avgProcessingSpeedInSec) { + avgHealth = HealthStatus.UnHealthy + } + } + const failure = await queryMetrics(dataset_id, prometheusQueries.validationFailure) + failure.health = getProcessingComponentHealth(failure, count, processingDefaultThreshold?.validationFailuresCount) + + const dedupFailure = await queryMetrics(dataset_id, prometheusQueries.dedupFailure) + dedupFailure.health = getProcessingComponentHealth(dedupFailure, count, processingDefaultThreshold?.dedupFailuresCount) + + const denormFailure = await queryMetrics(dataset_id, prometheusQueries.denormFailure) + denormFailure.health = getProcessingComponentHealth(denormFailure, count, processingDefaultThreshold?.denormFailureCount) + + const transformFailure = await queryMetrics(dataset_id, prometheusQueries.transformationFailure) + denormFailure.health = getProcessingComponentHealth(transformFailure, count, processingDefaultThreshold?.transformFailureCount) + + const components = [ + { + "type": "pipeline", + "status": flink + }, + { + "type": "eventsProcessedCount", + "count": count, + "status": health + }, + { + "type": "avgProcessingSpeedInSec", + "count": avgCount, + "status": avgHealth + }, + { + "type": "validationFailuresCount", + "count": failure?.count, + "status": failure?.health + }, + { + "type": "dedupFailuresCount", + "count": dedupFailure?.count, + "status": dedupFailure?.health + }, + { + "type": "denormFailureCount", + "count": denormFailure?.count, + "status": denormFailure?.health + }, + { + "type": "transformFailureCount", + "count": transformFailure?.count, + "status": transformFailure?.health + } + ] + const Healths = _.map(components, (component: any) => component?.status) + + const status = _.includes(Healths, HealthStatus.UnHealthy) ? HealthStatus.UnHealthy : HealthStatus.Healthy; + + return { components, status }; +} + +const getProcessingComponentHealth = (info: any, count: any, threshold: any) => { + let status = info.health + logger.debug({ info, count, threshold }) + if (info.health == HealthStatus.Healthy) { + if (info.count > 0 && count == 0) { + status = HealthStatus.UnHealthy + } else { + const percentage = (info.count / count) * 100 + if (percentage > threshold) { + status = HealthStatus.UnHealthy + } + } + } + return status; +} + +export const getQueryHealth = async (datasources: any, dataset: any): Promise<{ components: any, status: string }> => { + + const components: any = []; + const isMasterDataset = _.get(dataset, "type") == DatasetType.master; + let status = HealthStatus.Healthy; + if (!isMasterDataset) { + if (!_.isEmpty(datasources)) { + const druidTasks = await getDruidIndexerStatus(datasources); + components.push( + { + "type": "indexer", + "status": _.get(druidTasks, "status"), + "value": _.get(druidTasks, "value") + } + ) + if (_.get(druidTasks, "status") == HealthStatus.UnHealthy) { + status = HealthStatus.UnHealthy + } + } else { + components.push({ + "type": "indexer", + "status": HealthStatus.UnHealthy, + "value": [] + }) + status = HealthStatus.UnHealthy + } + } + + + const queriesCount = await queryMetrics(dataset?.dataset_id, prometheusQueries.queriesCount) + const defaultThresholds = await SystemConfig.getThresholds("query") + + components.push({ + "type": "queriesCount", + "count": queriesCount.count, + "status": queriesCount.health + }) + + const avgQueryReponseTimeInSec = await queryMetrics(dataset?.dataset_id, prometheusQueries.avgQueryResponseTimeInSec) + if (avgQueryReponseTimeInSec.count > defaultThresholds?.avgQueryReponseTimeInSec) { + avgQueryReponseTimeInSec.health = HealthStatus.UnHealthy + } + components.push({ + "type": "avgQueryReponseTimeInSec", + "count": avgQueryReponseTimeInSec.count, + "status": avgQueryReponseTimeInSec.health + }) + + const queriesFailed = await queryMetrics(dataset?.dataset_id, prometheusQueries.queriesFailedCount) + if (queriesCount.count == 0 && queriesFailed.count > 0) { + queriesFailed.health = HealthStatus.UnHealthy + } else { + const percentage = (queriesFailed.count / queriesCount.count) * 100; + if (percentage > defaultThresholds?.queriesFailed) { + queriesFailed.health = HealthStatus.UnHealthy + } + } + if ([queriesFailed.health, avgQueryReponseTimeInSec.health].includes(HealthStatus.UnHealthy)) { + status = HealthStatus.UnHealthy + } + + components.push({ + "type": "queriesFailed", + "count": queriesFailed.count, + "status": queriesFailed.health + }) + return { components, status } +} + +const getDruidIndexerStatus = async (datasources: any,) => { + try { + const results = await Promise.all(_.map(datasources, (datasource) => getDruidDataourceStatus(datasource["datasource"]))) + const values: any = [] + let status = HealthStatus.Healthy + _.forEach(results, (result: any) => { + logger.debug({ result }) + const sourceStatus = _.get(result, "payload.state") == "RUNNING" ? HealthStatus.Healthy : HealthStatus.UnHealthy + logger.debug({ sourceStatus }) + values.push( + { + "type": "druid", + "datasource": _.get(result, "id"), + "status": sourceStatus, + } + ) + if (sourceStatus == HealthStatus.UnHealthy) { + status = HealthStatus.UnHealthy + } + }) + return { value: values, status } + } catch (error) { + logger.error(error) + return { value: [], status: HealthStatus.UnHealthy } + } + + +} + +const getDruidDataourceStatus = async (datasourceId: string) => { + logger.debug(datasourceId) + const { data } = await druidHttpService.get(`/druid/indexer/v1/supervisor/${datasourceId}/status`) + return data; +} + +const getPostgresStatus = async (): Promise => { + try { + await postgresHealth() + } catch (error) { + logger.error(error) + return HealthStatus.UnHealthy + } + return HealthStatus.Healthy +} + +const connectToRedis = async (url: string) => { + return new Promise((resolve, reject) => { + createClient({ + url + }) + .on("error", (err: any) => { + reject(err) + }) + .on("connect", () => { + resolve("connected") + }) + .connect(); + }) +} + +const getRedisHealthStatus = async () => { + try { + await Promise.all([connectToRedis(`redis://${config.redis_config.denorm_redis_host}:${config.redis_config.denorm_redis_port}`), + connectToRedis(`redis://${config.redis_config.dedup_redis_host}:${config.redis_config.dedup_redis_port}`)]); + return HealthStatus.Healthy; + } catch (error) { + logger.error(error) + } + return HealthStatus.UnHealthy; +} + +const getKafkaHealthStatus = async () => { + try { + const status = await isKafkaHealthy() + return status ? HealthStatus.Healthy : HealthStatus.UnHealthy + } catch (error) { + return HealthStatus.UnHealthy + } + +} + +const getFlinkHealthStaus = async () => { + try { + const responses = await Promise.all( + [axios.get(config?.flink_job_configs?.masterdata_processor_job_manager_url as string + "/jobs"), + axios.get(config?.flink_job_configs?.pipeline_merged_job_manager_url as string + "/jobs")] + ) + const isHealthy = _.every(responses, (response: any) => { + const { data = {} } = response; + return _.get(data, "jobs[0].status") === "RUNNING" + }) + return isHealthy ? HealthStatus.Healthy : HealthStatus.UnHealthy; + } catch (error) { + logger.error("Unable to get flink status", error) + } + return HealthStatus.UnHealthy; +} + +const getDruidHealthStatus = async () => { + try { + const { data = false } = await druidHttpService.get("/status/health") + return data ? HealthStatus.Healthy : HealthStatus.UnHealthy + } catch (error) { + logger.error(error) + return HealthStatus.UnHealthy + } +} + +const getEventsProcessedToday = async (datasetId: string, isMasterDataset: boolean) => { + const startDate = moment().format(dateFormat); + const endDate = moment().add(1, "d").format(dateFormat); + const intervals = `${startDate}/${endDate}` + logger.debug({ datasetId, isMasterDataset }) + try { + const { data } = await executeNativeQuery({ + "queryType": "timeseries", + "dataSource": "system-events", + "intervals": intervals, + "granularity": { + "type": "all", + "timeZone": "Asia/Kolkata" + }, + "filter": { + "type": "and", + "fields": [ + { + "type": "selector", + "dimension": "ctx_module", + "value": "processing" + }, + { + "type": "selector", + "dimension": "ctx_dataset", + "value": datasetId + }, + { + "type": "selector", + "dimension": "ctx_pdata_id", + "value": isMasterDataset ? "MasterDataProcessorJob" : "DruidRouterJob" + }, + { + "type": "selector", + "dimension": "error_code", + "value": null + } + ] + }, + "aggregations": [ + { + "type": "longSum", + "name": "count", + "fieldName": "count" + } + ] + }) + return { health: HealthStatus.Healthy, count: _.get(data, "[0].result.count", 0) || 0 } + } catch (error) { + logger.error(error) + return { count: 0, health: HealthStatus.UnHealthy } + } +} + +const getAvgProcessingSpeedInSec = async (datasetId: string, isMasterDataset: boolean) => { + const startDate = moment().format(dateFormat); + const endDate = moment().add(1, "d").format(dateFormat); + const intervals = `${startDate}/${endDate}` + logger.debug({ datasetId, isMasterDataset }) + try { + const { data } = await executeNativeQuery({ + "queryType": "groupBy", + "dataSource": "system-events", + "intervals": intervals, + "granularity": { + "type": "all", + "timeZone": "Asia/Kolkata" + }, + "filter": { + "type": "and", + "fields": [ + { + "type": "selector", + "dimension": "ctx_module", + "value": "processing" + }, + { + "type": "selector", + "dimension": "ctx_dataset", + "value": datasetId + }, + { + "type": "selector", + "dimension": "ctx_pdata_id", + "value": isMasterDataset ? "MasterDataProcessorJob" : "DruidRouterJob" + }, + { + "type": "selector", + "dimension": "error_code", + "value": null + } + ] + }, + "aggregations": [ + { + "type": "longSum", + "name": "processing_time", + "fieldName": "total_processing_time" + }, + { + "type": "longSum", + "name": "count", + "fieldName": "count" + } + ], + "postAggregations": [ + { + "type": "expression", + "name": "average_processing_time", + "expression": "case_searched((count > 0),(processing_time/count),0", + } + ] + }) + logger.debug({ average_processing_time: JSON.stringify(data) }) + const count = _.get(data, "[0].event.average_processing_time", 0) || 0 + return { health: HealthStatus.Healthy, count: count / 1000 } + } catch (error) { + logger.error(error) + return { count: 0, health: HealthStatus.UnHealthy } + } +} \ No newline at end of file From e2e94a45945751606ac3b039c4f77942c98b70de Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Wed, 24 Jul 2024 22:16:30 +0530 Subject: [PATCH 056/235] #OBS-I1 Updated the imports and folders --- .../src/{v2 => }/controllers/DatasetHealth/DatasetHealth.ts | 2 +- .../DatasetHealth/DatasetHealthValidationSchema.json | 0 api-service/src/{v2 => }/services/DatasetHealthService.ts | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename api-service/src/{v2 => }/controllers/DatasetHealth/DatasetHealth.ts (98%) rename api-service/src/{v2 => }/controllers/DatasetHealth/DatasetHealthValidationSchema.json (100%) rename api-service/src/{v2 => }/services/DatasetHealthService.ts (100%) diff --git a/api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts b/api-service/src/controllers/DatasetHealth/DatasetHealth.ts similarity index 98% rename from api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts rename to api-service/src/controllers/DatasetHealth/DatasetHealth.ts index 610bfbc3..d523af1a 100644 --- a/api-service/src/v2/controllers/DatasetHealth/DatasetHealth.ts +++ b/api-service/src/controllers/DatasetHealth/DatasetHealth.ts @@ -4,7 +4,7 @@ import { schemaValidation } from "../../services/ValidationService"; import DatasetHealthRequestSchema from "./DatasetHealthValidationSchema.json" import { ResponseHandler } from "../../helpers/ResponseHandler"; import { DatasetStatus } from "../../types/DatasetModels"; -import { getDatasetHealth, getInfraHealth } from "../../services/HealthService"; +import { getDatasetHealth, getInfraHealth } from "../../services/DatasetHealthService"; import { obsrvError } from "../../types/ObsrvError"; import httpStatus from "http-status"; import { datasetService } from "../../services/DatasetService"; diff --git a/api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json b/api-service/src/controllers/DatasetHealth/DatasetHealthValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DatasetHealth/DatasetHealthValidationSchema.json rename to api-service/src/controllers/DatasetHealth/DatasetHealthValidationSchema.json diff --git a/api-service/src/v2/services/DatasetHealthService.ts b/api-service/src/services/DatasetHealthService.ts similarity index 100% rename from api-service/src/v2/services/DatasetHealthService.ts rename to api-service/src/services/DatasetHealthService.ts From a37d0bdad92fe666c766adb4d749632194457cb9 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Thu, 25 Jul 2024 09:58:04 +0530 Subject: [PATCH 057/235] #OBS-I2 updated dataset reset --- .../controllers/DatasetReset/DatasetReset.ts | 4 +-- .../DatasetResetValidationSchema.json | 0 .../DatasetStatusTransition.ts | 2 +- .../src/services/DatasetHealthService.ts | 30 +++++++++++++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) rename api-service/src/{v2 => }/controllers/DatasetReset/DatasetReset.ts (98%) rename api-service/src/{v2 => }/controllers/DatasetReset/DatasetResetValidationSchema.json (100%) diff --git a/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts b/api-service/src/controllers/DatasetReset/DatasetReset.ts similarity index 98% rename from api-service/src/v2/controllers/DatasetReset/DatasetReset.ts rename to api-service/src/controllers/DatasetReset/DatasetReset.ts index 29376ab2..29dd4844 100644 --- a/api-service/src/v2/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/controllers/DatasetReset/DatasetReset.ts @@ -7,7 +7,7 @@ import { ResponseHandler } from "../../helpers/ResponseHandler"; import { DatasetStatus, DatasetType, HealthStatus } from "../../types/DatasetModels"; import { Dataset } from "../../models/Dataset"; import logger from "../../logger"; -import { getDruidIndexers, getFlinkHealthStatus, restartDruidIndexers } from "../../services/HealthService"; +import { getDruidIndexers, getFlinkHealthStatus, restartDruidIndexers } from "../../services/DatasetHealthService"; import { Datasource } from "../../models/Datasource"; import { restartPipeline } from "../DatasetStatusTransition/DatasetStatusTransition"; @@ -48,7 +48,7 @@ const datasetReset = async (req: Request, res: Response) => { } as ErrorObject, req, res); } logger.debug(apiId, msgid, resmsgid, "dataset", dataset, category) - const isMasterDataset = _.get(dataset, "[0].type") == DatasetType.MasterDataset; + const isMasterDataset = _.get(dataset, "[0].type") == DatasetType.master; if(category == "processing") { const pipeLineStatus = await getFlinkHealthStatus() logger.debug({pipeLineStatus}) diff --git a/api-service/src/v2/controllers/DatasetReset/DatasetResetValidationSchema.json b/api-service/src/controllers/DatasetReset/DatasetResetValidationSchema.json similarity index 100% rename from api-service/src/v2/controllers/DatasetReset/DatasetResetValidationSchema.json rename to api-service/src/controllers/DatasetReset/DatasetResetValidationSchema.json diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 2d8d9ffb..a04c0868 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -221,7 +221,7 @@ const canRetireIfMasterDataset = async (dataset: Record) => { } } -const restartPipeline = async (dataset: Record) => { +export const restartPipeline = async (dataset: Record) => { return executeCommand(dataset.id, "RESTART_PIPELINE") } diff --git a/api-service/src/services/DatasetHealthService.ts b/api-service/src/services/DatasetHealthService.ts index b29fdd76..b3fb094c 100644 --- a/api-service/src/services/DatasetHealthService.ts +++ b/api-service/src/services/DatasetHealthService.ts @@ -481,3 +481,33 @@ const getAvgProcessingSpeedInSec = async (datasetId: string, isMasterDataset: bo return { count: 0, health: HealthStatus.UnHealthy } } } + +export const getDruidIndexers = async (datasources: any, status = HealthStatus.Healthy) => { + const results = await Promise.all(_.map(datasources, (datasource) => getDruidDataourceStatus(datasource["datasource"]))) + const indexers: any = [] + _.forEach(results, (result: any) => { + logger.debug({ result }) + const sourceStatus = _.get(result, "payload.state") == "RUNNING" ? HealthStatus.Healthy : HealthStatus.UnHealthy + logger.debug({ sourceStatus }) + if (sourceStatus == status) { + indexers.push( + { + "type": "druid", + "datasource": _.get(result, "id"), + "status": sourceStatus, + "state": _.get(result, "payload.state") + } + ) + } + }) + return indexers +} + +const restartDruidSupervisors = async (datasourceId: string) => { +const { data } = await druidHttpService.post(`/druid/indexer/v1/supervisor/${datasourceId}/resume`) +return data; +} + +export const restartDruidIndexers = async (datasources: any) => { + await Promise.all(_.map(datasources, (datasource) => restartDruidSupervisors(datasource["datasource"]))) +} From 5c6264fb2d12f254119e13237a35198bd624cd17 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 25 Jul 2024 11:09:04 +0530 Subject: [PATCH 058/235] #OBS-I116: fix: Command api and schema fixes --- .../DatasetCreate/DatasetCreate.ts | 4 ++- .../DatasetCreateValidationSchema.json | 14 +++++---- .../ReadyToPublishSchema.json | 14 +++++---- .../DatasetUpdate/DatasetUpdate.ts | 4 ++- .../DatasetUpdateValidationSchema.json | 3 +- command-service/src/command/db_command.py | 30 +++++++++---------- command-service/src/model/db_models.py | 6 ++-- 7 files changed, 42 insertions(+), 33 deletions(-) diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 0906098e..2867457b 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -69,9 +69,11 @@ const getDatasetConnectors = (connectorConfigs: Array>): Arr }) return _.map(uniqueConnectors, (config) => { return { + id: config.id, connector_id: config.connector_id, connector_config: cipherService.encrypt(JSON.stringify(config.connector_config)), - operations_config: config.operations_config + operations_config: config.operations_config, + version: config.version } }) } diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json b/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json index c8290408..13097a8b 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json +++ b/api-service/src/controllers/DatasetCreate/DatasetCreateValidationSchema.json @@ -71,8 +71,7 @@ }, "extraction_key": { "type": "string", - "default": "events", - "minLength": 1 + "default": "events" }, "dedup_config": { "type": "object", @@ -82,8 +81,7 @@ "default": false }, "dedup_key": { - "type": "string", - "minLength": 1 + "type": "string" } }, "if": { @@ -115,6 +113,11 @@ } }, "then": { + "properties": { + "extraction_key": { + "minLength": 1 + } + }, "required": ["extraction_key", "dedup_config"] } }, @@ -126,8 +129,7 @@ "default": true }, "dedup_key": { - "type": "string", - "minLength": 1 + "type": "string" } }, "if": { diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index 7a789977..281c62ac 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -376,7 +376,7 @@ "minLength": 1 }, "connector_config": { - "type": "object" + "type": "string" }, "operations_config": { "type": "object" @@ -402,11 +402,15 @@ "properties": { "dataset_config": { "properties": { - "data_key": { - "minLength": 1 + "keys_config": { + "properties": { + "data_key": { + "minLength": 1 + } + }, + "required": ["data_key"] } - }, - "required": ["data_key"] + } } } }, diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 1a8d68a5..25c9a633 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -121,9 +121,11 @@ const mergeConnectorsConfig = (currConfigs: any, newConfigs: any) => { return _.unionWith( _.map(addConfigs, (config) => { return { + id: config.id, connector_id: config.connector_id, connector_config: cipherService.encrypt(JSON.stringify(config.connector_config)), - operations_config: config.operations_config + operations_config: config.operations_config, + version: config.version } }), _.reject(currConfigs, (config) => { return _.includes(removeConfigs, config.connector_id)}), diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 7694474b..4dbc3e89 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -122,8 +122,7 @@ "default": true }, "dedup_key": { - "type": "string", - "minLength": 1 + "type": "string" } }, "if": { diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 55ed86dc..b8b256d3 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -171,8 +171,9 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): return result def _insert_connector_instances(self, dataset_id, draft_dataset_record): - emptyJson, result = {} - draft_connectors_config_record = draft_dataset_record.connectors_config + emptyJson = {} + result = {} + draft_connectors_config_record = draft_dataset_record.get('connectors_config') if draft_connectors_config_record is None: return result @@ -190,14 +191,14 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): '{connector_config.id}', '{dataset_id}', '{connector_config.connector_id}', - '{json.dumps(connector_config.connector_config).replace("'", "''")}', + '{connector_config.connector_config}', '{json.dumps(connector_config.operations_config).replace("'", "''")}', '{connector_config.data_format}', '{DatasetStatusType.Live.name}', '{json.dumps(emptyJson)}', '{json.dumps(emptyJson)}', - '{draft_dataset_record.created_by}', - '{draft_dataset_record.updated_by}', + '{draft_dataset_record.get('created_by')}', + '{draft_dataset_record.get('updated_by')}', '{current_timestamp}', '{current_timestamp}', '{current_timestamp}' @@ -206,7 +207,7 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): SET connector_config = '{json.dumps(connector_config.connector_config).replace("'", "''")}', operations_config = '{json.dumps(connector_config.operations_config).replace("'", "''")}', data_format = '{connector_config.data_format}', - updated_by = '{draft_dataset_record.updated_by}', + updated_by = '{draft_dataset_record.get('updated_by')}', updated_date = '{current_timestamp}', published_date = '{current_timestamp}', status = '{DatasetStatusType.Live.name}'; @@ -225,15 +226,15 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): '{connector_config.connector_id}', '{json.dumps(connector_config.connector_config).replace("'", "''")}', '{DatasetStatusType.Live.name}', - '{draft_dataset_record.created_by}', - '{draft_dataset_record.updated_by}', + '{draft_dataset_record.get('created_by')}', + '{draft_dataset_record.get('updated_by')}', '{current_timestamp}', '{current_timestamp}', '{current_timestamp}' ) ON CONFLICT (id) DO UPDATE SET connector_config = '{json.dumps(connector_config.connector_config).replace("'", "''")}', - updated_by = '{draft_dataset_record.updated_by}', + updated_by = '{draft_dataset_record.get('updated_by')}', updated_date = '{current_timestamp}', published_date = '{current_timestamp}', status = '{DatasetStatusType.Live.name}'; @@ -247,7 +248,7 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): - draft_dataset_transformations_record = draft_dataset_record.transformations_config + draft_dataset_transformations_record = draft_dataset_record.get('transformations_config') result = {} current_timestamp = dt.now() # Delete existing transformations @@ -263,7 +264,7 @@ def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): ) insert_query = f""" INSERT INTO dataset_transformations(id, dataset_id, field_key, transformation_function, - status, mode, created_by, updated_by, created_date, updated_date, published_date, metadata) + status, mode, created_by, updated_by, created_date, updated_date, published_date) VALUES ( '{dataset_id + '_' + transformation.field_key}', '{dataset_id}', @@ -271,12 +272,11 @@ def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): '{json.dumps(transformation.transformation_function).replace("'", "''")}', '{DatasetStatusType.Live.name}', '{transformation.mode}', - '{transformation.created_by}', - '{transformation.updated_by}', - '{current_timestamp}', + '{draft_dataset_record.get('created_by')}', + '{draft_dataset_record.get('updated_by')}', '{current_timestamp}', '{current_timestamp}', - '{json.dumps(transformation.metadata).replace("'", "''")}' + '{current_timestamp}' ) """ result = self.db_service.execute_upsert(insert_query) diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index 1d915d21..f77f8314 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -45,8 +45,8 @@ class DatasetsDraft: created_by: str created_date: datetime sample_data: dict | None = None - transformations_config: dict | None = None - connectors_config: dict | None = None + transformations_config: list[dict] | None = None + connectors_config: list[dict] | None = None denorm_config: dict | None = None tags: list[str] | None = None updated_by: str | None = None @@ -79,7 +79,7 @@ class DatasourcesDraft: class DatasetConnectorConfigDraft: id: str connector_id: str - connector_config: dict + connector_config: str | None version: str operations_config: dict | None = None data_format: str | None = 'json' From 3f500f0c14235f820eb1ec239f13386a65e83604 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 25 Jul 2024 11:49:35 +0530 Subject: [PATCH 059/235] #OBS-I116: fix: Command api fix in db query --- command-service/src/command/db_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index b8b256d3..6d2c7d46 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -191,7 +191,7 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): '{connector_config.id}', '{dataset_id}', '{connector_config.connector_id}', - '{connector_config.connector_config}', + '{json.dumps(connector_config.connector_config).replace("'", "''")}', '{json.dumps(connector_config.operations_config).replace("'", "''")}', '{connector_config.data_format}', '{DatasetStatusType.Live.name}', From c37df4deb1aae10773090ab15181910297100977 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Thu, 25 Jul 2024 15:52:39 +0530 Subject: [PATCH 060/235] #OBS-I1 Added Notifications and alerts APIs --- api-service/src/app.ts | 2 + api-service/src/configs/Config.ts | 5 + .../src/connections/grafanaConnection.ts | 10 + api-service/src/controllers/Alerts/Alerts.ts | 145 +++++++++ api-service/src/controllers/Alerts/Metric.ts | 86 +++++ api-service/src/controllers/Alerts/Silence.ts | 117 +++++++ .../NotificationChannel/Notification.ts | 122 +++++++ api-service/src/models/Alert.ts | 74 +++++ api-service/src/models/Metric.ts | 30 ++ api-service/src/models/Notification.ts | 46 +++ api-service/src/models/Silence.ts | 38 +++ api-service/src/routes/AlertsRouter.ts | 41 +++ api-service/src/routes/Router.ts | 1 + api-service/src/services/fs.ts | 17 + .../src/services/managers/constants.ts | 32 ++ .../managers/grafana/alert/helpers/index.ts | 302 ++++++++++++++++++ .../services/managers/grafana/alert/index.ts | 114 +++++++ .../src/services/managers/grafana/index.ts | 6 + .../grafana/notification/channels/email.ts | 53 +++ .../grafana/notification/channels/index.ts | 11 + .../grafana/notification/channels/slack.ts | 51 +++ .../grafana/notification/channels/teams.ts | 48 +++ .../grafana/notification/helpers/index.ts | 56 ++++ .../managers/grafana/notification/index.ts | 27 ++ .../grafana/notification/templates/index.ts | 9 + .../grafana/silences/helpers/index.ts | 50 +++ .../managers/grafana/silences/index.ts | 43 +++ api-service/src/services/managers/index.ts | 217 +++++++++++++ .../managers/prometheus/alert/index.ts | 17 + .../src/services/managers/prometheus/index.ts | 5 + .../managers/prometheus/notification/index.ts | 11 + .../managers/prometheus/silences/index.ts | 24 ++ api-service/src/types/AlertModels.ts | 36 +++ 33 files changed, 1846 insertions(+) create mode 100644 api-service/src/connections/grafanaConnection.ts create mode 100644 api-service/src/controllers/Alerts/Alerts.ts create mode 100644 api-service/src/controllers/Alerts/Metric.ts create mode 100644 api-service/src/controllers/Alerts/Silence.ts create mode 100644 api-service/src/controllers/NotificationChannel/Notification.ts create mode 100644 api-service/src/models/Alert.ts create mode 100644 api-service/src/models/Metric.ts create mode 100644 api-service/src/models/Notification.ts create mode 100644 api-service/src/models/Silence.ts create mode 100644 api-service/src/routes/AlertsRouter.ts create mode 100644 api-service/src/services/fs.ts create mode 100644 api-service/src/services/managers/constants.ts create mode 100644 api-service/src/services/managers/grafana/alert/helpers/index.ts create mode 100644 api-service/src/services/managers/grafana/alert/index.ts create mode 100644 api-service/src/services/managers/grafana/index.ts create mode 100644 api-service/src/services/managers/grafana/notification/channels/email.ts create mode 100644 api-service/src/services/managers/grafana/notification/channels/index.ts create mode 100644 api-service/src/services/managers/grafana/notification/channels/slack.ts create mode 100644 api-service/src/services/managers/grafana/notification/channels/teams.ts create mode 100644 api-service/src/services/managers/grafana/notification/helpers/index.ts create mode 100644 api-service/src/services/managers/grafana/notification/index.ts create mode 100644 api-service/src/services/managers/grafana/notification/templates/index.ts create mode 100644 api-service/src/services/managers/grafana/silences/helpers/index.ts create mode 100644 api-service/src/services/managers/grafana/silences/index.ts create mode 100644 api-service/src/services/managers/index.ts create mode 100644 api-service/src/services/managers/prometheus/alert/index.ts create mode 100644 api-service/src/services/managers/prometheus/index.ts create mode 100644 api-service/src/services/managers/prometheus/notification/index.ts create mode 100644 api-service/src/services/managers/prometheus/silences/index.ts create mode 100644 api-service/src/types/AlertModels.ts diff --git a/api-service/src/app.ts b/api-service/src/app.ts index ab864a98..d135e7bc 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -7,6 +7,7 @@ import bodyParser from "body-parser"; import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; +import { alertsRouter } from "./routes/AlertsRouter"; const app: Application = express(); @@ -17,6 +18,7 @@ app.use(errorHandler) app.use("/v2/", v2Router); app.use("/", druidProxyRouter); +app.use("/alerts/v1", alertsRouter); app.use("/", metricRouter); app.use("*", ResponseHandler.routeNotFound); app.use(obsrvErrorHandler); diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 4a9eb9ec..28f83b1e 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -107,5 +107,10 @@ export const config = { "encryption_config": { "encryption_key": process.env.encryption_key || "strong_encryption_key_to_encrypt", "encryption_algorithm": process.env.encryption_algorithm || "aes-256-ecb", + }, + "grafana_config": { + "dialect": process.env.dialet || 'postgres', + "url": process.env.grafana_url || "http://localhost:8000", + "access_token": process.env.grafana_token || "" } } diff --git a/api-service/src/connections/grafanaConnection.ts b/api-service/src/connections/grafanaConnection.ts new file mode 100644 index 00000000..76df0cea --- /dev/null +++ b/api-service/src/connections/grafanaConnection.ts @@ -0,0 +1,10 @@ +import axios from "axios"; +import { config } from "../configs/Config"; + +const grafanaHttpClient = axios.create({ + baseURL: config.grafana_config.url +}); + +grafanaHttpClient.defaults.headers.common['Authorization'] = config.grafana_config.access_token; + +export { grafanaHttpClient }; \ No newline at end of file diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts new file mode 100644 index 00000000..056a2475 --- /dev/null +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -0,0 +1,145 @@ +import { Request, Response, NextFunction } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { Alert } from "../../models/Alert"; +import httpStatus from "http-status"; +import errorResponse from "http-errors"; +import { deleteAlertRule, getAlertPayload, getAlertRule, getAlertsMetadata, publishAlert, retireAlertSilence, deleteSystemRules } from "../../services/managers"; +import _ from "lodash"; + +import { updateTelemetryAuditEvent } from "../../services/telemetry"; + +const telemetryObject = { type: "alert", ver: "1.0.0" }; + +const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const alertPayload = getAlertPayload(req.body); + const response = await Alert.create(alertPayload); + updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); + } catch (error: any) { + let errorMessage = _.get(error, 'message') + if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, 'parent.detail') + } + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const publishAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { alertId } = req.params; + const rulePayload: Record | null = await getAlertRule(alertId); + if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + if (rulePayload.status == "live") { + await deleteAlertRule(rulePayload, false); + } + await publishAlert(rulePayload); + updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); + } catch (error: any) { + console.log(error?.message) + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +}; + +const transformAlerts = async (alertModel: any) => { + const alert = alertModel?.toJSON(); + const status = _.get(alert, 'status'); + if (status !== "live") return alert; + return getAlertsMetadata(alert); +} + +const searchAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { limit, filters, offset, options = {} } = req.body?.request || {}; + const alerts = await Alert.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }), ...options }); + const alertRulesWithStatus = await Promise.all(_.map(alerts, transformAlerts)); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { alerts: alertRulesWithStatus, count: alerts.length } }); + } catch (error) { + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +}; + +const alertDetailsHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { alertId } = req.params; + const ruleModel: Record | null = await getAlertRule(alertId); + if (!ruleModel) { + return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + } + let rulePayload = ruleModel.toJSON(); + if (rulePayload.status == "live") { + rulePayload = await getAlertsMetadata(rulePayload) + } + updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { alerts: rulePayload } }); + } catch (error) { + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +} + +const deleteAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { alertId } = req.params; + const { hardDelete = "false" } = req.query + const ruleModel = await getAlertRule(alertId); + if (!ruleModel) { + return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + } + const rulePayload = ruleModel.toJSON(); + await deleteAlertRule(rulePayload, hardDelete === "true"); + updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); + } catch (error) { + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +} + +const updateAlertHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { alertId } = req.params; + const isEmpty = _.isEmpty(req.body); + if (isEmpty) throw new Error("Failed to update record"); + const ruleModel = await getAlertRule(alertId); + if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } + const rulePayload = ruleModel.toJSON(); + if (rulePayload.status == "live") { + await deleteAlertRule(rulePayload, false); + await retireAlertSilence(alertId); + } + const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); + await Alert.update({ ...updatedPayload, status: 'draft' }, { where: { id: alertId } }); + updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); + } catch (error: any) { + let errorMessage = _.get(error, 'message') + if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, 'parent.detail') + } + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const deleteSystemAlertsHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const body = req.body; + const { filters } = body; + if (!filters) throw new Error("Failed to update record"); + await deleteSystemRules({ filters, manager: "grafana" }); + await Alert.destroy({ where: filters }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: {} }); + } catch (error) { + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, (error).message))); + } +} + +export default { + alertDetailsHandler, + searchAlertHandler, + publishAlertHandler, + createAlertHandler, + deleteAlertHandler, + updateAlertHandler, + deleteSystemAlertsHandler +} + diff --git a/api-service/src/controllers/Alerts/Metric.ts b/api-service/src/controllers/Alerts/Metric.ts new file mode 100644 index 00000000..212d5b13 --- /dev/null +++ b/api-service/src/controllers/Alerts/Metric.ts @@ -0,0 +1,86 @@ +import _ from "lodash"; +import { Request, Response, NextFunction } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { Metrics } from "../../models/Metric"; +import httpStatus from "http-status"; +import errorResponse from "http-errors"; +import { updateTelemetryAuditEvent } from "../../services/telemetry"; + +const telemetryObject = { type: "metric", ver: "1.0.0" }; + +const createMetricHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { component } = req?.body; + const transformComponent = _.toLower(component); + const metricsBody = await Metrics.create({ ...(req.body), component: transformComponent }); + updateTelemetryAuditEvent({ request: req, object: { id: metricsBody?.dataValues?.id, ...telemetryObject } }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: metricsBody.dataValues.id } }); + } catch (error: any) { + let errorMessage = _.get(error, 'message') + if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, 'parent.detail') + } + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const listMetricsHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { limit, filters, offset } = _.get(req.body, "request") || {}; + const metricsPayload = await Metrics.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }) }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { metrics: metricsPayload, count: metricsPayload.length } }); + } catch (error) { + const errorMessage = _.get(error, 'message') + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const updateMetricHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const toUpdatePayload = req.body; + const { component } = toUpdatePayload; + const isEmpty = _.isEmpty(toUpdatePayload); + if (isEmpty) throw new Error("Failed to update record"); + const record = await Metrics.findOne({ where: { id } }); + if (!record) throw new Error(httpStatus[httpStatus.NOT_FOUND]); + updateTelemetryAuditEvent({ request: req, object: { id, ...telemetryObject }, currentRecord: record }) + await Metrics.update({ ...toUpdatePayload, ...(component) && { component: _.toLower(component) } }, { + where: { id } + }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } }); + } catch (error) { + let errorMessage = _.get(error, 'message') + if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, 'parent.detail') + } + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const deleteMetricHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { id } = req.params; + const record = await Metrics.findOne({ where: { id } }); + if (!record) throw new Error(httpStatus[httpStatus.NOT_FOUND]); + await record.destroy(); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } }); + } catch (error) { + const errorMessage = _.get(error, 'message') + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +const deleteMultipleMetricHandler = async (req: Request, res: Response, next: NextFunction) => { + try { + const { filters } = req.body; + if (!filters) throw new Error("Failed to update record"); + await Metrics.destroy({ where: filters }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: {} }); + } catch (error) { + const errorMessage = _.get(error, 'message') + next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) + } +} + +export default { createMetricHandler, listMetricsHandler, updateMetricHandler, deleteMetricHandler, deleteMultipleMetricHandler }; \ No newline at end of file diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts new file mode 100644 index 00000000..c578dcbc --- /dev/null +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -0,0 +1,117 @@ +import { Request, Response, NextFunction } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { createSilence, deleteSilence, getSilenceMetaData, updateSilence } from "../../services/managers"; +import httpStatus from "http-status"; +import errorResponse from "http-errors"; +import _ from "lodash"; +import { Silence } from "../../models/Silence"; +import { updateTelemetryAuditEvent } from "../../services/telemetry"; + +const telemetryObject = { type: "alert-silence", ver: "1.0.0" }; + +const createHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const payload = request.body; + const { startDate, endDate, alertId } = payload; + const existingSilence = await Silence.findOne({ where: { alert_id: alertId } }); + if (existingSilence) existingSilence.destroy(); + const grafanaResponse = await createSilence(payload); + if (!grafanaResponse) return next({ message: httpStatus[httpStatus.INTERNAL_SERVER_ERROR], statusCode: httpStatus.INTERNAL_SERVER_ERROR }) + + let start_date = new Date(startDate); + let end_date = new Date(endDate); + const silenceBody = { + id: grafanaResponse.silenceId, + manager: grafanaResponse.manager, + alert_id: alertId, + start_time: start_date, + end_time: end_date, + } + const sileneResponse = await Silence.create(silenceBody); + updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: sileneResponse.dataValues.id } }) + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const transformSilences = async (silenceModel: any) => { + const silence = silenceModel?.toJSON(); + return getSilenceMetaData(silence); +} + + +const listHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const silences = await Silence.findAll(); + const count = _.get(silences, 'length'); + const transformedSilences = await Promise.all(silences.map(transformSilences)); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { transformedSilences, ...(count && { count }) } }); + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const fetchHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const id = request.params.id; + const silenceModel = await Silence.findOne({ where: { id } }); + const transformedSilence = await transformSilences(silenceModel); + if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: transformedSilence }); + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const updateHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const id = request.params.id; + const payload = request.body; + const silenceModel = await Silence.findOne({ where: { id } }); + if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const silenceObject = silenceModel?.toJSON(); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: silenceObject }); + await updateSilence(silenceObject, payload); + const updatedStartTime = new Date(payload.startTime); + const updatedEndTime = new Date(payload.endTime); + const updatedSilence = { + ...silenceObject, + start_time: updatedStartTime, + end_time: updatedEndTime + } + const silenceResponse = await Silence.update(updatedSilence, { where: { id } }) + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } }) + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const deleteHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const id = request.params.id; + const silenceModel = await Silence.findOne({ where: { id } }); + if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const silenceObject = silenceModel?.toJSON(); + if (silenceObject?.status === 'expired') return next({ message: "Silence is already expired", statusCode: httpStatus.BAD_REQUEST }); + await deleteSilence(silenceObject); + await silenceModel.destroy(); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: silenceObject }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }) + } catch (err) { + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +export default { + createHandler, + listHandler, + fetchHandler, + updateHandler, + deleteHandler +} \ No newline at end of file diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts new file mode 100644 index 00000000..93c5a875 --- /dev/null +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -0,0 +1,122 @@ +import { Request, Response, NextFunction } from "express"; +import { Notification } from "../../models/Notification"; +import httpStatus from "http-status"; +import createError from "http-errors"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import { publishNotificationChannel, testNotificationChannel, updateNotificationChannel } from "../../services/managers"; +import _ from 'lodash'; +import { updateTelemetryAuditEvent } from "../../services/telemetry"; + +const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; + +const createHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const payload = request.body; + const notificationBody = await Notification.create(payload); + updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const updateHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { id } = request.params; + const updatedPayload = request.body; + const notificationPayloadModel = await Notification.findOne({ where: { id } }); + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); + if (_.get(notificationPayload, 'status') === "live") { + await updateNotificationChannel(notificationPayload); + } + await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const listHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { limit, filters, offset } = request.body?.request || {}; + const notifications = await Notification.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }) }); + const count = _.get(notifications, 'length'); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { notifications, ...(count && { count }) } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const fetchHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { id } = request.params; + const notificationPayloadModel = await Notification.findOne({ where: { id } }); + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayloadModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: notificationPayloadModel?.toJSON() }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const retireHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { id } = request.params; + const notificationPayloadModel = await Notification.findOne({ where: { id } }) + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); + await updateNotificationChannel(notificationPayload); + await Notification.update({ status: "retired" }, { where: { id } }) + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const publishHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { id } = request.params; + const notificationPayloadModel = await Notification.findOne({ where: { id } }) + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); + updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); + await publishNotificationChannel(notificationPayload); + Notification.update({ status: "live" }, { where: { id } }); + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +const testNotifationChannelHandler = async (request: Request, response: Response, next: NextFunction) => { + try { + const { message = "Hello Obsrv", payload = {} } = request.body; + const { id } = payload; + if (id) { + const notificationPayloadModel = await Notification.findOne({ where: { id } }) + const notificationPayload = notificationPayloadModel?.toJSON(); + if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + await testNotificationChannel({ ...notificationPayload, message }); + } + else { + await testNotificationChannel({ ...payload, message }) + } + ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "Notification Sent" } }); + } catch (err) { + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + next(error); + } +} + +export default { listHandler, createHandler, publishHandler, updateHandler, retireHandler, fetchHandler, testNotifationChannelHandler } \ No newline at end of file diff --git a/api-service/src/models/Alert.ts b/api-service/src/models/Alert.ts new file mode 100644 index 00000000..7f98cde5 --- /dev/null +++ b/api-service/src/models/Alert.ts @@ -0,0 +1,74 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; +import { v4 as uuidv4 } from 'uuid'; + +export const Alert = sequelize.define("alerts", { + id: { + type: DataTypes.STRING, + defaultValue: () => uuidv4().toString(), + primaryKey: true + }, + manager: { + type: DataTypes.STRING, + }, + name: { + type: DataTypes.STRING, + unique: true, + }, + status: { + type: DataTypes.ENUM("draft", "live", "retired"), + defaultValue: "draft", + }, + description: { + type: DataTypes.STRING, + }, + expression: { + type: DataTypes.STRING, + }, + severity: { + type: DataTypes.ENUM("warning", "critical"), + defaultValue: "warning" + }, + category: { + type: DataTypes.STRING, + }, + annotations: { + type: DataTypes.JSON, + defaultValue: {} + }, + labels: { + type: DataTypes.JSON, + defaultValue: {} + }, + frequency: { + type: DataTypes.STRING, + defaultValue: "1m", + }, + interval: { + type: DataTypes.STRING, + defaultValue: "1m", + }, + metadata: { + type: DataTypes.JSON, + defaultValue: {}, + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM", + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM", + }, + notification: { + type: DataTypes.JSON, + defaultValue: {}, + }, + context: { + type: DataTypes.JSON, + defaultValue: {}, + } +}, { + tableName: "alerts" +}); + diff --git a/api-service/src/models/Metric.ts b/api-service/src/models/Metric.ts new file mode 100644 index 00000000..605ef291 --- /dev/null +++ b/api-service/src/models/Metric.ts @@ -0,0 +1,30 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; +import { v4 as uuidv4 } from 'uuid'; + +export const Metrics = sequelize.define("metrics", { + id: { + type: DataTypes.STRING, + defaultValue: () => uuidv4().toString(), + primaryKey: true + }, + alias: { + type: DataTypes.STRING, + unique: true, + }, + component: { + type: DataTypes.STRING, + }, + subComponent: { + type: DataTypes.STRING, + }, + metric: { + type: DataTypes.STRING, + }, + context: { + type: DataTypes.JSON, + defaultValue: {} + } +}, { + tableName: "metrics" +}); \ No newline at end of file diff --git a/api-service/src/models/Notification.ts b/api-service/src/models/Notification.ts new file mode 100644 index 00000000..dc2bee5a --- /dev/null +++ b/api-service/src/models/Notification.ts @@ -0,0 +1,46 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; +import { v4 } from 'uuid'; + +export const Notification = sequelize.define("notificationchannel", { + id: { + type: DataTypes.STRING, + defaultValue: () => v4().toString(), + primaryKey: true + }, + manager: { + type: DataTypes.STRING + }, + name: { + type: DataTypes.STRING, + unique: true, + }, + status: { + type: DataTypes.ENUM("draft", "live", "retired"), + defaultValue: "draft", + }, + type: { + type: DataTypes.ENUM("Slack"), + defaultValue: "Slack", + }, + config: { + type: DataTypes.JSON, + defaultValue: {} + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM", + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "SYSTEM", + }, + context: { + type: DataTypes.JSON, + defaultValue: {}, + } +}, { + tableName: "notificationChannel" +}); + + diff --git a/api-service/src/models/Silence.ts b/api-service/src/models/Silence.ts new file mode 100644 index 00000000..3c272db1 --- /dev/null +++ b/api-service/src/models/Silence.ts @@ -0,0 +1,38 @@ +import { DataTypes } from "sequelize"; +import { sequelize } from "../connections/databaseConnection"; +import { v4 as uuid } from 'uuid'; + +export const Silence = sequelize.define("silences", { + id: { + type: DataTypes.STRING, + defaultValue: () => uuid().toString(), + primaryKey: true + }, + manager: { + type: DataTypes.STRING, + }, + start_time: { + type: DataTypes.DATE, + }, + end_time: { + type: DataTypes.DATE, + }, + alert_id: { + type: DataTypes.STRING, + }, + created_by: { + type: DataTypes.STRING, + defaultValue: "ADMIN", + }, + updated_by: { + type: DataTypes.STRING, + defaultValue: "ADMIN", + }, + context: { + type: DataTypes.JSON, + defaultValue: {}, + } +}, { + tableName: "silences", + timestamps: true, +}); \ No newline at end of file diff --git a/api-service/src/routes/AlertsRouter.ts b/api-service/src/routes/AlertsRouter.ts new file mode 100644 index 00000000..9efab170 --- /dev/null +++ b/api-service/src/routes/AlertsRouter.ts @@ -0,0 +1,41 @@ +import express from "express"; +import notificationHandler from '../controllers/NotificationChannel/Notification'; +import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; +import customAlertHandler from "../controllers/Alerts/Alerts"; +import metricAliasHandler from "../controllers/Alerts/Metric"; +import silenceHandler from "../controllers/Alerts/Silence"; + +export const alertsRouter = express.Router(); + +// Notifications + +alertsRouter.post("/notifications/search", setDataToRequestObject("api.alert.notification.list"), notificationHandler.listHandler); +alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), notificationHandler.createHandler); +alertsRouter.get("/notifications/publish/:id", setDataToRequestObject("api.alert.notification.publish"), notificationHandler.publishHandler); +alertsRouter.post("/notifications/test", setDataToRequestObject("api.alert.notification.test"), notificationHandler.testNotifationChannelHandler); +alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), notificationHandler.updateHandler); +alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), notificationHandler.retireHandler); +alertsRouter.get("/notifications/get/:id", setDataToRequestObject("api.alert.notification.get"), notificationHandler.fetchHandler); + +// alerts +alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), customAlertHandler.createAlertHandler); +alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), customAlertHandler.publishAlertHandler); +alertsRouter.post(`/search`, setDataToRequestObject("api.alert.list"), customAlertHandler.searchAlertHandler); +alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), customAlertHandler.alertDetailsHandler); +alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteAlertHandler); +alertsRouter.delete("/delete", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteSystemAlertsHandler); +alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), customAlertHandler.updateAlertHandler); + +// metrics +alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), metricAliasHandler.createMetricHandler); +alertsRouter.post("/metric/alias/search", setDataToRequestObject("api.metric.list"), metricAliasHandler.listMetricsHandler); +alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"),metricAliasHandler.updateMetricHandler); +alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"),metricAliasHandler.deleteMetricHandler); +alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), metricAliasHandler.deleteMultipleMetricHandler); + +// silence +alertsRouter.post("/silence/create",setDataToRequestObject("api.alert.silence.create"),silenceHandler.createHandler); +alertsRouter.get("/silence/search",setDataToRequestObject("api.alert.silence.list"),silenceHandler.listHandler); +alertsRouter.get("/silence/get/:id",setDataToRequestObject("api.alert.silence.get"),silenceHandler.fetchHandler); +alertsRouter.patch("/silence/update/:id",setDataToRequestObject("api.alert.silence.edit"),silenceHandler.updateHandler); +alertsRouter.delete("/silence/delete/:id",setDataToRequestObject("api.alert.silence.delete"),silenceHandler.deleteHandler); \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index ec64cc94..569e3eac 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -43,5 +43,6 @@ router.post("/datasets/status-transition", setDataToRequestObject("api.datasets. router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); + //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file diff --git a/api-service/src/services/fs.ts b/api-service/src/services/fs.ts new file mode 100644 index 00000000..e08597d6 --- /dev/null +++ b/api-service/src/services/fs.ts @@ -0,0 +1,17 @@ +import fs from 'fs'; +import path from 'path'; + +export const scrapModules = (folderPath: string, basename: string) => { + const mapping = new Map>(); + fs.readdirSync(folderPath) + .filter((file) => file !== basename) + .map((file) => { + const { + default: { name, ...others }, + } = require(path.join(folderPath, file)) as { default: Type }; + + mapping.set(name, others); + }); + + return mapping; +}; diff --git a/api-service/src/services/managers/constants.ts b/api-service/src/services/managers/constants.ts new file mode 100644 index 00000000..d6d9c028 --- /dev/null +++ b/api-service/src/services/managers/constants.ts @@ -0,0 +1,32 @@ +export default { + "RECORD_ALREADY_PUBLISHED": "Record is already published", + "RECORD_NOT_FOUND": "Record does not exist in the system", + "PROMETHEUS_DATASOURCE_NOT_FOUND": "Prometheus data source does not exist", + "RULE_ALREADY_EXIST": "Alert Rule already exists", + "RULE_ALREADY_PUBLISHED": "Alert Rule already published", + "INVALID_MANAGER": "Invalid Alert manager", + "UPDATE_FAILURE": "Failed to update record", + "INVALID_QUERY": "Invalid Query", + "RECORD_ALREADY_DELETED": "Record is already deleted", + "FOLDER_NOT_FOUND": "Alert folder not found", + "CATEGORY_NOT_EXIST": "Category does not exists", + "UPDATE_STATUS_FAILURE": "Failed to update alert status", + "WEBHOOK_MISSING": "webhook url is missing", + "INTERNAL_SERVER_ERROR": "Internal Server Error", + "DEFAULT_TEST_NOTIFICATION_MESSAGE": "Hello World", + "METHOD_NOT_IMPLEMENTED": "method not implemented", + "SILENCE_EXPIRED": "Silence is already expired", + "ALERTS_NOT_RETIRED": "Failed to retire alerts for dataset", + "ALERTS_FETCH_FAILURE": "Failed to fetch alerts for dataset", + "ALERTS_RETIRED_SUCCESSFULLY": "Alerts retired successfully for datasets", + "SILENCE_DELETED_SUCCESSFULLY":"Silence deleted successfully for the Alert rule", + "ALERTS_PUBLISHED_SUCCESSFULLY": "Alerts published successfully for datasets", + "ALERT_PUBLISH_FAILURE":"Failed to publish alerts for dataset", + "ALERT_CREATE_FAILURE":"Failed to create alerts for datasets", + "METRIC_ALIAS_CREATE_FAILURE":"Failed to create metric alias for datasets", + "ALERTS_NOT_FOUND":"Alert rule does not exist", + "METRIC_ALIAS_NOT_FOUND":"Metric alias does not exist", + "METRIC_ALIAS_FETCH_FAILURE":"Failed to fetch alert metrics for dataset", + "METRIC_ALIAS_NOT_DELETED": "Failed to delete matric alias for datasets", + "METRIC_ALIAS_DELETED_SUCCESSFULLY":"Metric alias deleted successfully for datasets" +} diff --git a/api-service/src/services/managers/grafana/alert/helpers/index.ts b/api-service/src/services/managers/grafana/alert/helpers/index.ts new file mode 100644 index 00000000..e9c19e29 --- /dev/null +++ b/api-service/src/services/managers/grafana/alert/helpers/index.ts @@ -0,0 +1,302 @@ +import _ from "lodash"; + +import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; +import constants from "../../../constants"; +import { Notification } from "../../../../../models/Notification"; + +export const getRules = () => { + return grafanaHttpClient.get("/api/ruler/grafana/api/v1/rules"); +}; + +const getDatasource = () => { + return grafanaHttpClient.get("api/datasources"); +}; + +const alerts = () => { + return grafanaHttpClient.get("/api/prometheus/grafana/api/v1/rules"); +}; + +const deleteAlertRule = (alertCategory: string) => { + return grafanaHttpClient.delete(`/api/ruler/grafana/api/v1/rules/${alertCategory}/${alertCategory}`); +}; + +const getFilteredAlerts = () => { + return grafanaHttpClient.get("/api/alertmanager/grafana/api/v2/alerts?silenced&active&inhibited"); +} + +const deleteAlertFolder = async (folderName: string) => { + const folderUID = await getFolderUid(folderName); + if (!folderUID) throw new Error(constants.FOLDER_NOT_FOUND); + return grafanaHttpClient.delete(`/api/folders/${folderUID}`) +}; + +const checkIfGroupNameExists = async (category: string) => { + const response = await getRules(); + const rules = _.get(response, 'data'); + if(!_.has(rules, category)) return undefined; + return _.find(_.flatMap(_.values(rules)), { + name: category, + }); +}; + +const checkIfRuleExists = (category: string | any, ruleName: string) => { + const ruleExists = category.rules.some((rule: any) => rule.grafana_alert.title === ruleName); + if (ruleExists) { + throw new Error(constants.RULE_ALREADY_EXIST); + } +}; + +const getPrometheusDataSource = async () => { + const dataSources = await getDatasource(); + const prometheusDataSource = _.find(dataSources.data, { type: "prometheus" }); + if (!prometheusDataSource) { + throw new Error(constants.PROMETHEUS_DATASOURCE_NOT_FOUND); + } + return prometheusDataSource; +}; + +const getSpecificRule = async (payload: Record) => { + const alertrules = await alerts(); + const groups = _.get(alertrules, "data.data.groups"); + const ruleGroup = _.find(groups, (group: any) => group.name == payload.category); + return _.find(_.get(ruleGroup, 'rules'), (rule: any) => rule.name == payload.name); +}; + +const updateMetadata = (metadata: any, dataSource: string, expression: string) => { + const str = JSON.stringify(metadata); + metadata = _.replace(_.replace(str, /\$datasourceUid/g, dataSource), /\$expr/g, expression); + metadata = JSON.parse(metadata); + return metadata; +}; + +const addGrafanaRule = (payload: Record) => { + return grafanaHttpClient.post(`/api/ruler/grafana/api/v1/rules/${payload.name}`, payload); +}; + +const getFolders = () => { + return grafanaHttpClient.get("/api/folders") +}; + +const getFolderUid = async (name: string) => { + const folders = await getFolders() + const folderPayload = _.find(folders.data, (folder: any) => folder.title == name) || []; + return folderPayload.uid; +} +const createFolder = (title: string) => { + const payload = { title }; + return grafanaHttpClient.post("/api/folders", payload); +}; + +const createFolderIfNotExists = async (folderName: string) => { + const folders = await getFolders(); + const isExists = _.find(folders.data, folder => _.get(folder, 'title') === folderName); + if (isExists) return; + return createFolder(folderName); +} + +const getQueryModel = (payload: Record) => { + const { metric, operator, threshold } = payload; + const relativeTimeRange = { "from": 600, "to": 0 }; + + return [ + { + "refId": "A", + "datasourceUid": "$datasourceUid", + "queryType": "", + "relativeTimeRange": relativeTimeRange, + "model": { + "refId": "A", + "hide": false, + "editorMode": "code", + "expr": metric, + "legendFormat": "__auto", + "range": true + } + }, + { + "refId": "B", + "datasourceUid": "__expr__", + "queryType": "", + "model": { + "refId": "B", + "hide": false, + "type": "reduce", + "datasource": { + "uid": "__expr__", + "type": "__expr__" + }, + "settings": { + "mode": "replaceNN", + "replaceWithValue": 0 + }, + "conditions": [ + { + "type": "query", + "evaluator": { + "params": [], + "type": operator + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B" + ] + }, + "reducer": { + "params": [], + "type": "last" + } + } + ], + "reducer": "last", + "expression": "A" + }, + "relativeTimeRange": relativeTimeRange + }, + { + "refId": "C", + "datasourceUid": "__expr__", + "queryType": "", + "model": { + "refId": "C", + "hide": false, + "type": "threshold", + "datasource": { + "uid": "__expr__", + "type": "__expr__" + }, + "conditions": [ + { + "type": "query", + "evaluator": { + "params": threshold, + "type": operator + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "C" + ] + }, + "reducer": { + "params": [], + "type": "last" + } + } + ], + "expression": "B" + }, + "relativeTimeRange": relativeTimeRange + } + ] +} + +const queryOperators = [ + { + label: "greater than (>)", + value: "gt", + symbol: ">" + }, + { + label: "less than (<)", + value: "lt", + symbol: "<" + }, + { + "label": "within range", + "value": "within_range", + "symbol": "within_range" + } +]; + +const getQueryExpression = (payload: Record) => { + const { metric, operator, threshold } = payload; + const operatorSymbol = _.get(_.find(queryOperators, operatorMetadata => _.get(operatorMetadata, 'value') === operator), 'symbol'); + return `(${metric}) ${operatorSymbol} ${threshold}`; +} + +const getMatchingLabels = async (channels: string[]) => { + try { + + const fetchChannel = (id: string) => { + return Notification.findOne({ where: { id } }) + .then(response => response?.toJSON()) + .then(channelMetadata => { + const { name, type } = channelMetadata; + return `notificationChannel_${name}_${type}`; + }) + .catch(err => null); + } + + const matchingLabels = await Promise.all(channels.map(fetchChannel)); + return _.reduce(_.compact(matchingLabels), (acc, value) => { + return { ...acc, [value]: "true" }; + }, {}) + + } catch (error) { + return {} + } +} + +const transformRule = async ({ value, condition, metadata, isGroup }: any) => { + const { name, id, interval, category, frequency, labels = {}, annotations = {}, severity, description, notification = {} } = value; + const annotationObj = { ...annotations, description: description }; + const channels = _.get(notification, 'channels') || []; + const matchingLabelsForNotification = await getMatchingLabels(channels); + + const payload = { + grafana_alert: { + title: name, + condition: condition, + no_data_state: _.get(metadata, 'no_data_state', 'NoData'), + exec_err_state: _.get(metadata, 'exec_err_state', 'Error'), + data: metadata, + is_paused: false, + }, + for: interval, + annotations: annotationObj, + labels: { + 'alertId': id, + ...labels, + ...(severity && { severity }), + ...matchingLabelsForNotification + } + }; + + if (isGroup) { + return { name: category, interval: frequency, rules: [payload] }; + } + + return payload; +}; + + +const groupRulesByCategory = (payload: Record[]) => { + return _.reduce(payload, (accumulator: Record, current: Record) => { + const { category, name } = current; + const existing = _.get(accumulator, category) || []; + accumulator[category] = [...existing, name]; + return accumulator + }, {}); +} + +export { + getPrometheusDataSource, + updateMetadata, + checkIfGroupNameExists, + checkIfRuleExists, + transformRule, + createFolderIfNotExists, + addGrafanaRule, + getSpecificRule, + deleteAlertRule, + deleteAlertFolder, + getQueryExpression, + getQueryModel, + getFilteredAlerts, + groupRulesByCategory +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/alert/index.ts b/api-service/src/services/managers/grafana/alert/index.ts new file mode 100644 index 00000000..b2dcd1e2 --- /dev/null +++ b/api-service/src/services/managers/grafana/alert/index.ts @@ -0,0 +1,114 @@ +import _ from "lodash"; +import { addGrafanaRule, checkIfGroupNameExists, checkIfRuleExists, createFolderIfNotExists, deleteAlertFolder, deleteAlertRule, getPrometheusDataSource, getQueryExpression, getQueryModel, getSpecificRule, transformRule, updateMetadata, getFilteredAlerts, getRules, groupRulesByCategory } from "./helpers"; +import { Silence } from "../../../../models/Silence"; +import constants from "../../constants"; + +const publishAlert = async (payload: Record) => { + let metaData = payload.metadata || []; + const conditionRef: any = _.last(metaData.query); + if (!conditionRef) throw new Error(constants.INVALID_QUERY); + const prometheusDataSource = await getPrometheusDataSource(); + metaData = await updateMetadata(metaData, _.get(prometheusDataSource, "uid"), payload.expression); + const categoryExists = await checkIfGroupNameExists(payload.category); + let alertRulePayload; + const transformPayload = { value: payload, condition: conditionRef.refId, metadata: metaData.query }; + if (categoryExists) { + checkIfRuleExists(categoryExists, payload.name); + const newRule = await transformRule({ ...transformPayload, isGroup: false }); + const transformRules = _.concat(categoryExists.rules, newRule); + alertRulePayload = { ...categoryExists, rules: transformRules }; + } else { + await createFolderIfNotExists(payload.category); + alertRulePayload = await transformRule({ ...transformPayload, isGroup: true }); + } + + return addGrafanaRule(alertRulePayload); +}; + +const getAlerts = async (payload: Record) => { + const context = payload?.context || {}; + const alertId = _.get(payload, 'id'); + const { err, alertData } = await getSpecificRule(payload) + .then(alertData => { + if (!alertData) throw new Error() + return { err: null, alertData } + }) + .catch(err => ({ err: err?.message || "rule not found in alerting manager", alertData: null })); + + const silenceModel = await Silence.findOne({ where: { alert_id: alertId } }); + const silenceData = silenceModel?.toJSON(); + let silenceState: Record = { state: '', silenceId: '' }; + + if (silenceData) { + const { end_time } = silenceData; + const currentTime = new Date().getTime(); + const endTime = new Date(end_time).getTime(); + if (currentTime < endTime) { + silenceState.state = 'muted'; + silenceState['endTime'] = endTime; + } else { + silenceState.state = 'unmuted'; + } + silenceState.silenceId = silenceData.id; + } else { + silenceState.state = 'unmuted'; + } + + return { ...payload, context: { ...context, err }, ...(alertData && { alertData }), silenceState }; +}; + +const deleteAlert = async (payload: Record) => { + const { name, category } = payload; + const alertCategory = await checkIfGroupNameExists(category); + if (!alertCategory) throw new Error(constants.CATEGORY_NOT_EXIST); + + if (_.get(alertCategory, 'rules.length') > 1) { + const filteredRule = _.filter(alertCategory.rules, (rule) => _.get(rule, 'grafana_alert.title') !== name) || []; + const filteredGroup = { ...alertCategory, rules: filteredRule }; + return addGrafanaRule(filteredGroup); + } + + await deleteAlertRule(category); + return deleteAlertFolder(category); +} + +const generateAlertPayload = (payload: Record) => { + if (!(_.get(payload, "metadata.queryBuilderContext"))) return payload; + const { metadata } = payload; + const { queryBuilderContext } = metadata; + const expression = getQueryExpression(queryBuilderContext); + const queryModel = getQueryModel(queryBuilderContext); + const updatedMetadata = { ...metadata, query: queryModel } + return { ...payload, expression, metadata: updatedMetadata } +} + +const filterSystemRulesPredicate = (rule: Record) => { + const labels = _.get(rule, 'labels') || {}; + const isSystemAlert = _.find(labels, (value, key) => (key === "alertSource" && value === "system-rule-ingestor-job")); + if (!isSystemAlert) return true; + return false; +} + +const deleteSystemRules = async (filters: Record) => { + const response = await getRules(); + const existingRules = _.cloneDeep(_.get(response, 'data')) as Record[]>; + for (const [category, evaluationGroups] of Object.entries(existingRules)) { + const evaluationGroup = _.find(evaluationGroups, ["name", category]); + if (!evaluationGroup) continue; + const rules = _.get(evaluationGroup, 'rules') || [] + const filteredRules = _.filter(rules, filterSystemRulesPredicate); + try { + if (_.isEmpty(filteredRules)) { + await deleteAlertRule(category); + await deleteAlertFolder(category); + } else { + const filteredGroup = { ...evaluationGroup, rules: filteredRules }; + await addGrafanaRule(filteredGroup); + } + } catch (err) { + console.log(err) + } + } +} + +export { publishAlert, generateAlertPayload, deleteAlert, getAlerts, deleteSystemRules } \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/index.ts b/api-service/src/services/managers/grafana/index.ts new file mode 100644 index 00000000..30c2b82d --- /dev/null +++ b/api-service/src/services/managers/grafana/index.ts @@ -0,0 +1,6 @@ +import * as alertFunctions from './alert' +import * as notificationFunctions from './notification'; +import * as silenceFunctions from './silences'; + +const service = { name: "grafana", ...alertFunctions, ...notificationFunctions, ...silenceFunctions }; +export default service \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/email.ts b/api-service/src/services/managers/grafana/notification/channels/email.ts new file mode 100644 index 00000000..d4628517 --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/channels/email.ts @@ -0,0 +1,53 @@ +import _ from 'lodash'; +import { IChannelConfig } from "../../../../../types/AlertModels"; +import { grafanaHttpClient } from '../../../../../connections/grafanaConnection'; + +const getReceiverObject = ({ name, recipientAddresses, message, multipleAddresses, type }: any) => { + return { + name, + grafana_managed_receiver_configs: [ + { + settings: { + addresses: recipientAddresses, + subject: "{{template \"email\" .}}", + singleEmail: !multipleAddresses, + ...(message && { + message + }) + }, + secureSettings: {}, + type: type, + name, + disableResolveMessage: false + } + ] + } +} + +const service: IChannelConfig = { + name: "email", + service: { + generateConfigPayload(payload: Record): Record { + const { type, config, name } = payload; + const { recipientAddresses, message, subject = "Obsrv Alert", labels = [[`notificationChannel_${name}_${type.toLowerCase()}`, "=", "true"]] } = config; + const multipleAddresses = _.size(_.split(recipientAddresses, ';')) > 1; + return { + notificationPolicy: { + receiver: name, + object_matchers: labels + }, + receiver: getReceiverObject({ name, type, message, multipleAddresses, recipientAddresses, subject }) + } + }, + testChannel(payload: Record): Promise { + const { name, type, config, message = "Test Channel" } = payload; + const { recipientAddresses, subject = "Obsrv Alert" } = config; + const multipleAddresses = _.size(_.split(recipientAddresses, ';')) > 1; + const alert = { annotations: { description: message }, labels: {} }; + const body = { alert, receivers: [getReceiverObject({ name, type, multipleAddresses, recipientAddresses, subject })] }; + return grafanaHttpClient.post("api/alertmanager/grafana/config/api/v1/receivers/test", body); + } + } +} + +export default service; \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/index.ts b/api-service/src/services/managers/grafana/notification/channels/index.ts new file mode 100644 index 00000000..e7f5f87f --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/channels/index.ts @@ -0,0 +1,11 @@ +import path from 'path'; +import { scrapModules } from '../../../../../services/fs'; +import { IChannelConfig } from '../../../../../types/AlertModels'; + +const channels = scrapModules(__dirname, path.basename(__filename)); + +export const getChannelService = (channelName: string) => { + const channel = channels.get(channelName.toLowerCase()); + if (!channel) throw new Error('invalid channel'); + return channel; +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/slack.ts b/api-service/src/services/managers/grafana/notification/channels/slack.ts new file mode 100644 index 00000000..2da034fc --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/channels/slack.ts @@ -0,0 +1,51 @@ +import axios from 'axios'; +import _ from 'lodash'; +import CONSTANTS from '../../../constants' +import { IChannelConfig } from '../../../../../types/AlertModels'; + +const generateConfigPayload = (payload: Record): Record => { + const { type, config, name } = payload; + const { webhookUrl, labels = [[`notificationChannel_${name}_${type.toLowerCase()}`, "=", "true"]] } = config; + + return { + notificationPolicy: { + receiver: name, + object_matchers: labels + }, + receiver: { + name, + grafana_managed_receiver_configs: [ + { + settings: { + title: "{{ template \"slack_title\" . }}", + text: "{{ template \"slack_body\" . }}" + }, + secureSettings: { + token: "", + url: webhookUrl + }, + type, + name, + disableResolveMessage: false + } + ] + } + } +} + +const testChannel = (payload: Record): Promise => { + const { config, message } = payload; + const { webhookUrl } = config; + if (!webhookUrl) throw new Error(CONSTANTS.WEBHOOK_MISSING); + return axios.post(webhookUrl, { text: message }, { headers: { "Content-Type": "application/json" } }) +} + +const service: IChannelConfig = { + name: "slack", + service: { + generateConfigPayload, + testChannel + } +} + +export default service \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/teams.ts b/api-service/src/services/managers/grafana/notification/channels/teams.ts new file mode 100644 index 00000000..5cc0e689 --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/channels/teams.ts @@ -0,0 +1,48 @@ +import axios from 'axios'; +import _ from 'lodash'; +import CONSTANTS from '../../../constants'; +import { IChannelConfig } from '../../../../../types/AlertModels'; + +const generateConfigPayload = (payload: Record): Record => { + const {type, config, name} = payload; + const { webhookUrl, labels = [[`notificationChannel_${name}_${type.toLowerCase()}`, "=", "true"]] } = config; + return { + notificationPolicy: { + receiver: name, + object_matchers: labels, + }, + receiver: { + name, + grafana_managed_receiver_configs: [ + { + settings: { + url: webhookUrl, + title: "{{template \"teams.alerts.title\" .}}", + text: "{{template \"teams.alerts.message\" .}}" + }, + secureSettings: {}, + name, + type, + disableResolveMessage: false, + } + ] + } + } +} + +const testChannel = (payload: Record): Promise => { + const {config, message} = payload; + const {webhookUrl} = config; + if(!webhookUrl) throw new Error(CONSTANTS.WEBHOOK_MISSING) + return axios.post(webhookUrl, {text: message}, {headers: {"Content-Type": "application/json"}}) +} + +const service: IChannelConfig = { + name: "teams", + service: { + generateConfigPayload, + testChannel + } +} + +export default service; \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/helpers/index.ts b/api-service/src/services/managers/grafana/notification/helpers/index.ts new file mode 100644 index 00000000..a90c2ad7 --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/helpers/index.ts @@ -0,0 +1,56 @@ +import _ from "lodash"; +import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; +import { getChannelService } from '../channels'; +import defaultTemplates from '../templates'; + +const generateChannelConfig = (payload: Record) => { + const { type } = payload; + const channel = getChannelService(type); + return channel.service.generateConfigPayload(payload); +}; + +const getAlertManagerConfig = async () => { + return grafanaHttpClient.get("/api/alertmanager/grafana/config/api/v1/alerts") + .then(response => response.data); +}; + +const updateAlertManagerConfig = async (payload: Record) => { + return grafanaHttpClient.post("/api/alertmanager/grafana/config/api/v1/alerts", payload); +}; + +const getReceivers = (alertmanager_config: Record) => _.get(alertmanager_config, 'alertmanager_config.receivers') as Array; +const getRoutes = (alertmanager_config: Record) => _.get(alertmanager_config, 'alertmanager_config.route.routes') as Array; +const getTemplates = (alertmanager_config: Record) => _.get(alertmanager_config, 'template_files') as Record; + +const createContactPointsAndNotificationPolicy = async (metadata: Record) => { + const { receiver, notificationPolicy } = metadata; + const config = await getAlertManagerConfig(); + const existingReceivers = getReceivers(config) || []; + const existingRoutes = getRoutes(config) || []; + const existingTemplates = getTemplates(config) || {}; + _.set(config, 'alertmanager_config.receivers', [...existingReceivers, receiver]); + _.set(config, 'alertmanager_config.route.routes', [...existingRoutes, notificationPolicy]); + _.set(config, 'template_files', { ...existingTemplates, ...defaultTemplates }); + return updateAlertManagerConfig(config); +}; + +const removeReceiverAndNotificationPolicy = (payload: Record, alertManagerConfig: Record) => { + const { name } = payload; + const clonedAlertManagerConfig = _.cloneDeep(alertManagerConfig); + const existingReceivers = getReceivers(clonedAlertManagerConfig) || []; + const existingRoutes = getRoutes(clonedAlertManagerConfig) || []; + _.remove(existingRoutes, route => _.get(route, 'receiver') === name); + _.remove(existingReceivers, receiver => _.get(receiver, 'name') === name); + return clonedAlertManagerConfig; +}; + +export { + generateChannelConfig, + getAlertManagerConfig, + updateAlertManagerConfig, + getReceivers, + getRoutes, + getTemplates, + createContactPointsAndNotificationPolicy, + removeReceiverAndNotificationPolicy +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/index.ts b/api-service/src/services/managers/grafana/notification/index.ts new file mode 100644 index 00000000..bf7f387b --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/index.ts @@ -0,0 +1,27 @@ +import _ from "lodash"; +import { getChannelService } from './channels'; +import { createContactPointsAndNotificationPolicy, generateChannelConfig, getAlertManagerConfig, removeReceiverAndNotificationPolicy, updateAlertManagerConfig } from "./helpers"; + +const updateNotificationChannel = async (payload: Record) => { + const alertManagerConfig = await getAlertManagerConfig(); + const updatedAlertManagerConfig = removeReceiverAndNotificationPolicy(payload, alertManagerConfig); + return updateAlertManagerConfig(updatedAlertManagerConfig); +}; + +const createNotificationChannel = (payload: Record) => { + const { type, config, name } = payload; + const { notificationPolicy, receiver } = generateChannelConfig({ type, config, name }); + return createContactPointsAndNotificationPolicy({ notificationPolicy, receiver }); +}; + +const testNotificationChannel = async (payload: Record) => { + const { type } = payload; + const channel = getChannelService(type); + return channel.service.testChannel(payload); +}; + +export { + updateNotificationChannel, + createNotificationChannel, + testNotificationChannel +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/templates/index.ts b/api-service/src/services/managers/grafana/notification/templates/index.ts new file mode 100644 index 00000000..cf00f1c0 --- /dev/null +++ b/api-service/src/services/managers/grafana/notification/templates/index.ts @@ -0,0 +1,9 @@ +const templates = { + "email": "{{ define \"email\" }}\n{{ range .Alerts.Firing }}\nObsrv - {{.Labels.alertname}}\n{{ end }}\n{{ end }}", + "slack_title": "{{ define \"slack_title\" }}\n[{{ .Status | toUpper }}: {{ if eq .Status \"firing\" }}{{ .Alerts.Firing | len }}{{ else if eq .Status \"resolved\" }}{{.Alerts.Resolved | len }}{{ end }}] {{ .GroupLabels.SortedPairs.Values | join \" \" }}\n{{ end }}", + "slack_body": "{{ define \"slack_body\" -}}\n{{- $status := \"resolved\" -}}\n{{ range $index, $alert := .Alerts }}\n{{ if eq $index 0 }}\n Severity: {{ .Labels.severity }}\n Description: {{ .Annotations.description }}\n Status: {{ .Status }}\n{{ end }}\n{{ end }}\n{{ end }}", + "teams.alerts.title": "{{ define \"teams.alerts.title\" }}\n{{ .CommonLabels.alertname }}\n{{ end }}", + "teams.alerts.message": "{{ define \"teams.alerts.message\" }}\n\n{{ if gt (len .Alerts.Firing) 0 }} **Firing**\n{{ template \"__teams_text_alert_list\" .Alerts.Firing }}{{ if gt (len .Alerts.Resolved) 0 }}\n\n{{ end }}\n{{ end }}\n{{ if gt (len .Alerts.Resolved) 0 }}**Resolved**\n{{ template \"__teams_text_alert_list\" .Alerts.Resolved }}{{ end }}{{ end }}\n\n{{ define \"__teams_text_alert_list\" }}\n{{ range . }}\n\n{{ if gt (len .Labels.severity) 0 }}\nSeverity = {{ .Labels.severity }}\n{{ end }}\nDescription = {{ .Annotations.description }}\n{{ end }}\n{{ end }}" +} + +export default templates; \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/silences/helpers/index.ts b/api-service/src/services/managers/grafana/silences/helpers/index.ts new file mode 100644 index 00000000..599ab05f --- /dev/null +++ b/api-service/src/services/managers/grafana/silences/helpers/index.ts @@ -0,0 +1,50 @@ +import _ from "lodash"; +import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; + +const addSilence = (payload: Record) => { + return grafanaHttpClient.post("/api/alertmanager/grafana/api/v2/silences", payload); +} + +const fetchAllSilences = () => { + return grafanaHttpClient.get("/api/alertmanager/grafana/api/v2/silences"); +} + +const getSilence = (silenceId: string) => { + return grafanaHttpClient.get(`/api/alertmanager/grafana/api/v2/silence/${silenceId}`); +} + +const disableSilence = (silenceId: string) => { + return grafanaHttpClient.delete(`/api/alertmanager/grafana/api/v2/silence/${silenceId}`); +} + +const getCurrentSilenceStatus = async (silenceId: string) => { + const response = await getSilence(silenceId); + const currentSilenceStatus = _.get(response, 'data.status.state'); + return currentSilenceStatus; +} + +const transfromPayload = (alertId: string, startDate: string, endDate: string, silenceId?: string,) => { + return { + "comment": "Silence for alert: " + alertId, + "createdby": "admin", + "startsat": startDate, + "endsat": endDate, + "id": silenceId, + "matchers": [ + { + name: "alertId", + value: alertId, + isRegex: false, + isEqual: true + } + ] + } +} + +export { + addSilence, + fetchAllSilences, + disableSilence, + getCurrentSilenceStatus, + transfromPayload +} \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/silences/index.ts b/api-service/src/services/managers/grafana/silences/index.ts new file mode 100644 index 00000000..95e23b21 --- /dev/null +++ b/api-service/src/services/managers/grafana/silences/index.ts @@ -0,0 +1,43 @@ +import _ from "lodash"; +import { addSilence, disableSilence, getCurrentSilenceStatus, transfromPayload } from "./helpers"; + +const createSilence = async (payload: Record) => { + const { alertId, startDate, endDate, manager } = payload; + const grafanaPayload = transfromPayload(alertId, startDate, endDate); + + try { + const addSilenceResponse = await addSilence(grafanaPayload); + const silenceId = addSilenceResponse.data.silenceID; + + return { + silenceId, + manager, + } + + } catch (err: any) { + throw new Error(err); + } +} + +const getSilenceMetadata = async (payload: Record) => { + const silenceId = _.get(payload, "id") + const silenceStatus = await getCurrentSilenceStatus(silenceId); + return { + ...payload, + status: silenceStatus + } +} + +const updateSilence = async (silence: Record, payload: Record) => { + const { id, alert_id } = silence; + const { startTime, endTime } = payload; + const grafanaPayload = transfromPayload(alert_id, startTime, endTime, id); + await addSilence(grafanaPayload); +} + + +const deleteSilence = (silenceId: string) => { + return disableSilence(silenceId); +} + +export { createSilence, getSilenceMetadata, updateSilence, deleteSilence } \ No newline at end of file diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts new file mode 100644 index 00000000..e2e57568 --- /dev/null +++ b/api-service/src/services/managers/index.ts @@ -0,0 +1,217 @@ +import _ from "lodash"; +import { Alert } from "../../models/Alert"; +import grafanaService from "./grafana/index"; +import prometheusService from "./prometheus/index"; +import { Silence } from "../../models/Silence"; +import { Metrics } from "../../models/Metric"; +import constants from "./constants"; + +export const getAlertRule = (id: string) => { + return Alert.findOne({ where: { id } }); +} + +const getService = (manager: string) => { + switch (manager) { + case "grafana": { + return grafanaService; + } + case "prometheus": { + return prometheusService; + } + default: + throw new Error("Invalid Alert manager"); + } +}; + +export const publishAlert = async (payload: Record) => { + const { id, manager } = payload; + const service = getService(manager); + const publishResponse = await service.publishAlert(payload) + await updateStatus(id, "live"); + return publishResponse; +}; + + +const updateStatus = (id: string, status: string) => { + return Alert.update({ status }, { where: { id } }); +} + +const deleteRule = (id: string) => { + return Alert.destroy({ where: { id } }) +} + +export const deleteAlertRule = async (payload: Record, hardDelete: boolean) => { + const { id, manager, status } = payload; + + if (status == "live") { + try { + const service = getService(manager); + await service.deleteAlert(payload) + } catch (err: any) { + console.log(err) + } + } + + if (hardDelete) { + return deleteRule(id); + } + + return updateStatus(id, "retired"); +} + + +export const deleteSystemRules = async (payload: Record) => { + const { rules = [], manager } = payload; + const service = getService(manager); + return service.deleteSystemRules(rules); +} + +export const getAlertsMetadata = (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.getAlerts(payload); +} + +export const getAlertPayload = (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.generateAlertPayload(payload); +} + +export const publishNotificationChannel = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.createNotificationChannel(payload); +} + +export const testNotificationChannel = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.testNotificationChannel(payload); +} + +export const updateNotificationChannel = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.updateNotificationChannel(payload); +} + +export const createSilence = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.createSilence(payload); +} + +export const getSilenceMetaData = async (payload: Record) => { + const { manager } = payload; + const service = getService(manager); + return service.getSilenceMetadata(payload); +} + +export const updateSilence = async (silence: Record, payload: Record) => { + const { manager } = silence; + const service = getService(manager); + await service.updateSilence(silence, payload); +} + +export const deleteSilence = async (payload: Record) => { + const { id, manager } = payload; + const service = getService(manager); + await service.deleteSilence(id); +} + +export const deleteAlertByDataset = async (payload: Record) => { + try { + const { name } = payload; + const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) + if (!alertRulePayload) throw new Error(constants.ALERTS_NOT_FOUND) + for (let payload of alertRulePayload) { + await deleteAlertRule(payload, true) + await retireAlertSilence(_.get(payload, "id") || "") + } + return constants.ALERTS_RETIRED_SUCCESSFULLY; + } catch (error: any) { + throw new Error(constants.ALERTS_NOT_RETIRED); + } +} + +export const deleteMetricAliasByDataset = async (payload: Record) => { + try { + const { name } = payload; + const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name } }) + if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) + for (let payload of metricAliasPayload) { + await payload.destroy() + } + return constants.METRIC_ALIAS_DELETED_SUCCESSFULLY; + } catch (error: any) { + throw new Error(constants.METRIC_ALIAS_NOT_DELETED); + } +} + +export const getAlertByDataset = async (payload: Record) => { + try { + const { name } = payload; + const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) + if (!alertRulePayload) throw new Error(constants.ALERTS_NOT_FOUND) + return alertRulePayload; + } catch (error) { + throw new Error(constants.ALERTS_FETCH_FAILURE); + } +} + +export const getAlertMetricsByDataset = async (payload: Record) => { + try { + const { name } = payload; + const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name }, raw: true }) + if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) + return metricAliasPayload; + } catch (error: any) { + throw new Error(constants.METRIC_ALIAS_NOT_DELETED); + } +} + +export const createAlertsByDataset = async (payload: any) => { + try { + for (let alerts of payload) { + const alertPayload = _.omit(alerts as any, ["id", "status", "createdAt", "updatedAt", "created_by", "updated_by"]) + await Alert.create(alertPayload) + } + } catch (error) { + throw new Error(constants.ALERT_CREATE_FAILURE); + } +} + +export const createMetricAliasByDataset = async (payload: any) => { + try { + for (let metrics of payload) { + const metricsPayload = _.omit(metrics as any, ["id", "createdAt", "updatedAt"]) + await Metrics.create(metricsPayload) + } + } catch (error) { + throw new Error(constants.METRIC_ALIAS_CREATE_FAILURE); + } +} + +export const publishAlertByDataset = async (payload: Record) => { + try { + const { name } = payload; + const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) + if (!alertRulePayload) throw new Error("Alert rule does not exist") + for (let payload of alertRulePayload) { + await publishAlert(payload) + } + return constants.ALERTS_PUBLISHED_SUCCESSFULLY; + } catch (error: any) { + throw new Error(constants.ALERT_PUBLISH_FAILURE); + } +} + +export const retireAlertSilence = async (alert_id: string) => { + const silencePayload = await Silence.findOne({ where: { alert_id }, raw: true }); + if (silencePayload) { + await deleteSilence(silencePayload); + await Silence.destroy({ where: { alert_id } }); + } + return constants.SILENCE_DELETED_SUCCESSFULLY; +} \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/alert/index.ts b/api-service/src/services/managers/prometheus/alert/index.ts new file mode 100644 index 00000000..f2bd3a2f --- /dev/null +++ b/api-service/src/services/managers/prometheus/alert/index.ts @@ -0,0 +1,17 @@ +import CONSTANTS from '../../constants' + +export const publishAlert = async (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} +export const getAlerts = async (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} +export const deleteAlert = async (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} +export const generateAlertPayload = (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} +export const deleteSystemRules= async (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} diff --git a/api-service/src/services/managers/prometheus/index.ts b/api-service/src/services/managers/prometheus/index.ts new file mode 100644 index 00000000..c8c2e7df --- /dev/null +++ b/api-service/src/services/managers/prometheus/index.ts @@ -0,0 +1,5 @@ +import * as alertFunctions from './alert' +import * as notificationFunctions from './notification'; +import * as silenceFunctions from './silences'; + +export default { ...alertFunctions, ...notificationFunctions, ...silenceFunctions } \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/notification/index.ts b/api-service/src/services/managers/prometheus/notification/index.ts new file mode 100644 index 00000000..91b663cb --- /dev/null +++ b/api-service/src/services/managers/prometheus/notification/index.ts @@ -0,0 +1,11 @@ +import CONSTANTS from '../../constants' + +export const createNotificationChannel = (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} +export const testNotificationChannel = async (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} +export const updateNotificationChannel = (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/silences/index.ts b/api-service/src/services/managers/prometheus/silences/index.ts new file mode 100644 index 00000000..1c073881 --- /dev/null +++ b/api-service/src/services/managers/prometheus/silences/index.ts @@ -0,0 +1,24 @@ +import CONSTANTS from '../../constants'; + +const createSilence = async (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} + +const getSilenceMetadata = async (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) +} + +const updateSilence = async (silence: Record, payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} + +const deleteSilence = async (payload: Record) => { + throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); +} + +export { + createSilence, + getSilenceMetadata, + updateSilence, + deleteSilence +} \ No newline at end of file diff --git a/api-service/src/types/AlertModels.ts b/api-service/src/types/AlertModels.ts new file mode 100644 index 00000000..ae741db6 --- /dev/null +++ b/api-service/src/types/AlertModels.ts @@ -0,0 +1,36 @@ +type IPromiseAny = Promise; + +interface IAlert { + publishAlert(payload: Record): IPromiseAny; + getAlerts(payload: Record): IPromiseAny; + deleteAlert(payload: Record): IPromiseAny; +} + +interface INotificationChannel { + createNotificationChannel(payload: Record): IPromiseAny + testNotificationChannel(payload: Record, message: string): IPromiseAny; + updateNotificationChannel(payload: Record): IPromiseAny; +} + +interface IManager extends IAlert, INotificationChannel { + name: string +} + +export interface IGrafana extends IManager { + name: "grafana" +} + +export interface IPrometheus extends IManager { + name: "prometheus" +} + +interface IChannelService { + generateConfigPayload: (payload: Record) => Record, + testChannel: (payload: Record) => Promise +} + +export interface IChannelConfig { + name: string, + service: IChannelService +} + From 44fe81b2f328ccee8e584e4889bf047f74a42254 Mon Sep 17 00:00:00 2001 From: yashashk Date: Fri, 26 Jul 2024 10:45:49 +0530 Subject: [PATCH 061/235] #OBS-I138: added decrypted response for the read api for connectors_config field and added defaults updated date and created date to list api --- .../src/controllers/DatasetList/DatasetList.ts | 2 +- .../src/controllers/DatasetRead/DatasetRead.ts | 12 ++++++++++-- .../DatasetStatusTransition.ts | 11 +++++++---- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/api-service/src/controllers/DatasetList/DatasetList.ts b/api-service/src/controllers/DatasetList/DatasetList.ts index 81b0056e..82acbf62 100644 --- a/api-service/src/controllers/DatasetList/DatasetList.ts +++ b/api-service/src/controllers/DatasetList/DatasetList.ts @@ -12,7 +12,7 @@ export const apiId = "api.datasets.list" export const errorCode = "DATASET_LIST_FAILURE" const liveDatasetStatus = ["Live", "Retired", "Purged"] const draftDatasetStatus = ["Draft", "ReadyToPublish"] -const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config"] +const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config", "created_date", "updated_date"] const datasetList = async (req: Request, res: Response) => { diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 610deca8..2ecc875d 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -5,6 +5,7 @@ import { ResponseHandler } from "../../helpers/ResponseHandler"; import { DatasetDraft } from "../../models/DatasetDraft"; import { datasetService } from "../../services/DatasetService"; import { obsrvError } from "../../types/ObsrvError"; +import { cipherService } from "../../services/CipherService"; export const apiId = "api.datasets.read"; export const errorCode = "DATASET_READ_FAILURE" @@ -16,7 +17,7 @@ const validateRequest = (req: Request) => { const { dataset_id } = req.params; const fields = req.query.fields; - if(fields && typeof fields !== 'string') { + if (fields && typeof fields !== 'string') { throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS_VAL", `The specified fields [${fields}] in the query param is not a string.`, "BAD_REQUEST", 400); } const fieldValues = fields ? _.split(fields, ",") : []; @@ -34,7 +35,14 @@ const datasetRead = async (req: Request, res: Response) => { const { fields, mode } = req.query; const attributes = !fields ? defaultFields : _.split(fields, ","); const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes) - if(!dataset) { + if (dataset.connectors_config) { + dataset.connectors_config = dataset.connectors_config.map((connector: any) => ({ + ...connector, + connector_config: JSON.parse(cipherService.decrypt(connector.connector_config)) + })); + } + + if (!dataset) { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } else { ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 2d8d9ffb..8a091e6e 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -24,7 +24,7 @@ const allowedTransitions: Record = { } const liveDatasetActions = ["Retire", "Archive", "Purge"] -const validateRequest = (req: Request, datasetId: any) => { +const validateRequest = (req: Request, datasetId: any) => { const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) if (!isRequestValid.isValid) { throw obsrvError(datasetId, invalidRequest, isRequestValid.message, "BAD_REQUEST", 400) @@ -91,8 +91,11 @@ const deleteDataset = async (dataset: Record) => { const readyForPublish = async (dataset: Record) => { - - const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + + let draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) + defaultConfigs = _.omit(defaultConfigs, ["router_config"]) + _.merge(draftDataset, defaultConfigs) const datasetValid: Record = schemaValidation(draftDataset, ReadyToPublishSchema) if (!datasetValid.isValid) { throw { @@ -180,7 +183,7 @@ const updateMasterDataConfig = async (draftDataset: Record) => { if(draftDataset.type === 'master') { if(draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() - if(_.isEmpty(results)) { + if (_.isEmpty(results)) { throw { code: "REDIS_DB_INDEX_FETCH_FAILED", message: `Unable to fetch the redis db index for the master data`, From 6399a8026d39010087eaf4665ec07cfc7fe77acf Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 26 Jul 2024 11:14:19 +0530 Subject: [PATCH 062/235] #OBS-I116: fix: feat: Dataset copy and export api implementation --- .../controllers/DatasetCopy/DatasetCopy.ts | 53 +++++++++++++++++ .../DatasetCopy/DatasetCopyHelper.ts | 30 ++++++++++ .../DatasetCopy/RequestValidationSchema.json | 57 +++++++++++++++++++ .../DatasetExport/DatasetExport.ts | 51 +++++++++++++++++ api-service/src/routes/Router.ts | 5 +- api-service/src/services/DatasetService.ts | 46 ++++++++++----- 6 files changed, 227 insertions(+), 15 deletions(-) create mode 100644 api-service/src/controllers/DatasetCopy/DatasetCopy.ts create mode 100644 api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts create mode 100644 api-service/src/controllers/DatasetCopy/RequestValidationSchema.json create mode 100644 api-service/src/controllers/DatasetExport/DatasetExport.ts diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts new file mode 100644 index 00000000..8f86eeed --- /dev/null +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -0,0 +1,53 @@ + +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import * as _ from "lodash"; +import { schemaValidation } from "../../services/ValidationService"; +import validationSchema from "./RequestValidationSchema.json"; +import { datasetService, getLiveDatasetConfigs } from "../../services/DatasetService"; +import { updateRecords } from "./DatasetCopyHelper"; +import { obsrvError } from "../../types/ObsrvError"; + +export const apiId = "api.dataset.copy"; + +const validateRequest = (req: Request) => { + + const isValidSchema = schemaValidation(req.body, validationSchema); + if (!isValidSchema?.isValid) { + throw obsrvError("", "DATASET_COPY_INVALID_INPUT", isValidSchema.message, "BAD_REQUEST", 400); + } +} + +const fetchDataset = async (req: Request, newDatasetId: string) => { + const datasetId = _.get(req, "body.request.source.datasetId"); + const isLive = _.get(req, "body.request.source.isLive"); + + let dataset = isLive ? await getLiveDatasetConfigs(datasetId) : await datasetService.getDraftDataset(datasetId) + if (dataset === null) { + throw obsrvError(datasetId, "DATASET_NOT_EXISTS", `Dataset ${datasetId} does not exists`, "NOT_FOUND", 404); + } + + if (_.get(dataset, "api_version") != "v2") { + const migratedConfigs = await datasetService.migrateDatasetV1(datasetId, dataset) + dataset = { ...dataset, ...migratedConfigs } + } + + return dataset; +} + +const datasetCopy = async (req: Request, res: Response) => { + + validateRequest(req); + const newDatasetId = _.get(req, "body.request.destination.datasetId"); + const dataset = await fetchDataset(req, newDatasetId); + updateRecords(dataset, newDatasetId) + const response = await datasetService.createDraftDataset(dataset).catch(err => { + if (err?.name === 'SequelizeUniqueConstraintError') { + throw obsrvError(newDatasetId, "DATASET_ALREADY_EXISTS", `Dataset with id ${newDatasetId} already exists`, "BAD_REQUEST", 400); + } + throw obsrvError(newDatasetId, "DATASET_COPY_FAILURE", `Failed to clone dataset`, "INTERNAL_SERVER_ERROR", 500); + }); + return ResponseHandler.successResponse(req, res, { status: 200, data: { dataset_id: _.get(response, "id"), message: `Dataset clone successful` } }); +} + +export default datasetCopy; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts new file mode 100644 index 00000000..7de9c953 --- /dev/null +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -0,0 +1,30 @@ +import * as _ from "lodash"; +import { DatasetStatus } from "../../types/DatasetModels"; +import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +import { query } from "../../connections/databaseConnection"; +import { config } from "../../configs/Config"; +const version = defaultDatasetConfig.version; + +export const updateRecords = (datasetRecord: Record, newDatasetId: string): void => { + const dataset_id = newDatasetId; + _.set(datasetRecord, 'api_version', "v2") + _.set(datasetRecord, 'status', DatasetStatus.Draft) + _.set(datasetRecord, "dataset_id", dataset_id) + _.set(datasetRecord, "id", dataset_id) + _.set(datasetRecord, "name", dataset_id) + _.set(datasetRecord, "version_key", Date.now().toString()) + _.set(datasetRecord, 'version', version); + _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) + _.set(datasetRecord, "router_config", { topic: newDatasetId }) + + if (datasetRecord?.type === "master") { + _.set(datasetRecord, "dataset_config.cache_config.redis_db", updateMasterDatasetConfig(datasetRecord?.dataset_config)) + } +} + +const updateMasterDatasetConfig = async (datasetConfig: any) => { + let nextRedisDB = datasetConfig.redis_db; + const { results }: any = await query("SELECT nextval('redis_db_index')") + if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; + return _.assign(datasetConfig, { "redis_db": nextRedisDB }) +} diff --git a/api-service/src/controllers/DatasetCopy/RequestValidationSchema.json b/api-service/src/controllers/DatasetCopy/RequestValidationSchema.json new file mode 100644 index 00000000..3109df60 --- /dev/null +++ b/api-service/src/controllers/DatasetCopy/RequestValidationSchema.json @@ -0,0 +1,57 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasets.copy"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "source": { + "type": "object", + "properties": { + "datasetId": { + "type": "string", + "minLength": 3 + }, + "isLive": { + "type": "boolean" + } + }, + "required": ["datasetId", "isLive"] + }, + "destination": { + "type": "object", + "properties": { + "datasetId": { + "type": "string", + "minLength": 3 + } + }, + "required": ["datasetId"] + } + }, + "required": ["source", "destination"], + "additionalProperties": false + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false +} diff --git a/api-service/src/controllers/DatasetExport/DatasetExport.ts b/api-service/src/controllers/DatasetExport/DatasetExport.ts new file mode 100644 index 00000000..23f54d41 --- /dev/null +++ b/api-service/src/controllers/DatasetExport/DatasetExport.ts @@ -0,0 +1,51 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import { DatasetStatus } from "../../types/DatasetModels"; +import { datasetService, getLiveDatasetConfigs } from "../../services/DatasetService"; +import _ from "lodash"; +import { obsrvError } from "../../types/ObsrvError"; + +const validateDataset = async (req: Request) => { + + const { dataset_id } = req.params; + const { status = DatasetStatus.Live } = req.query; + + let datasetRecord = status == DatasetStatus.Live ? await getLiveDatasetConfigs(dataset_id) : await datasetService.getDraftDataset(dataset_id) + if (_.isEmpty(datasetRecord)) { + throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found to export`, "NOT_FOUND", 404); + } + + const datasetStatus = _.get(datasetRecord, "status") + if (_.includes([DatasetStatus.Draft, DatasetStatus.Retired, DatasetStatus.Archived], datasetStatus)) { + throw obsrvError(dataset_id, "DATASET_EXPORT_FAILURE", `Dataset with status:${datasetStatus} cannot be exported`, "BAD_REQUEST", 400); + } + + if (_.get(datasetRecord, "api_version") != "v2") { + const migratedConfigs = await datasetService.migrateDatasetV1(dataset_id, datasetRecord) + datasetRecord = { ...datasetRecord, ...migratedConfigs } + } + return datasetRecord; +} + +const datasetExport = async (req: Request, res: Response) => { + + const datasetRecord = await validateDataset(req) + const updateDataset = transformConnectorsConfig(datasetRecord) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: updateDataset }); + +} + +const transformConnectorsConfig = (dataset: Record) => { + const connectorConfig = _.get(dataset, "connectors_config") + if (!_.isEmpty(connectorConfig)) { + const updatedConnectorConfig = _.map(connectorConfig, config => { + const { id, connector_id, operations_config, version } = config + return { id, connector_id, connector_config: {}, operations_config, version } + }) + return { ...dataset, connectors_config: updatedConnectorConfig } + } + return dataset +} + +export default datasetExport; \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 569e3eac..299a9e4c 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -21,6 +21,8 @@ import { sqlQuery } from "../controllers/QueryWrapper/SqlQueryWrapper"; import DatasetStatusTansition from "../controllers/DatasetStatusTransition/DatasetStatusTransition"; import datasetHealth from "../controllers/DatasetHealth/DatasetHealth"; import DataSchemaGenerator from "../controllers/GenerateDataSchema/GenerateDataSchema"; +import DatasetExport from "../controllers/DatasetExport/DatasetExport"; +import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; export const router = express.Router(); @@ -42,7 +44,8 @@ router.post("/files/generate-url", setDataToRequestObject("api.files.generate-ur router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); - +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), DatasetCopy); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 84b6e2cd..da6ccef1 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -68,7 +68,7 @@ class DatasetService { return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); } - getConnectors = async (dataset_id: string, attributes?: string[]) => { + getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); } @@ -93,12 +93,26 @@ class DatasetService { } migrateDraftDataset = async (datasetId: string, dataset: Record): Promise => { + const dataset_id = _.get(dataset, "id") + const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); + const transaction = await sequelize.transaction(); + try { + await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); + await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); + await DatasetSourceConfigDraft.destroy({ where: { dataset_id }, transaction }); + await transaction.commit(); + } catch (err) { + await transaction.rollback(); + throw err; + } + return await this.getDraftDataset(datasetId); + } + migrateDatasetV1= async (dataset_id: string, dataset: Record): Promise => { let draftDataset: Record = { api_version: "v2", version_key: Date.now().toString() } - const dataset_id = _.get(dataset, "id") const dataset_config: any = _.get(dataset, "dataset_config"); draftDataset["dataset_config"] = { indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, @@ -124,18 +138,7 @@ class DatasetService { version: "v1" } }) - - const transaction = await sequelize.transaction(); - try { - await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); - await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); - await DatasetSourceConfigDraft.destroy({ where: { dataset_id }, transaction }); - await transaction.commit(); - } catch (err) { - await transaction.rollback(); - throw err; - } - return await this.getDraftDataset(datasetId); + return draftDataset; } private getTransformationCategory = (section: string):string => { @@ -319,4 +322,19 @@ class DatasetService { } +export const getLiveDatasetConfigs = async (dataset_id: string) => { + + let datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) + const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) + const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) + + if(!_.isEmpty(transformations)){ + datasetRecord["transformations_config"] = transformations + } + if(!_.isEmpty(connectors)){ + datasetRecord["connectors_config"] = connectors + } + return datasetRecord; +} + export const datasetService = new DatasetService(); \ No newline at end of file From 8c8221e5bac21d4cda419338233f7aa475a2669a Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Fri, 26 Jul 2024 11:17:29 +0530 Subject: [PATCH 063/235] #OBS-I2 Refactoring as per v2 APIs --- .../controllers/DatasetReset/DatasetReset.ts | 106 ++++++------------ 1 file changed, 34 insertions(+), 72 deletions(-) diff --git a/api-service/src/controllers/DatasetReset/DatasetReset.ts b/api-service/src/controllers/DatasetReset/DatasetReset.ts index 29dd4844..68b272f1 100644 --- a/api-service/src/controllers/DatasetReset/DatasetReset.ts +++ b/api-service/src/controllers/DatasetReset/DatasetReset.ts @@ -2,94 +2,56 @@ import { Request, Response } from "express"; import _ from "lodash"; import { schemaValidation } from "../../services/ValidationService"; import DatasetResetRequestSchema from "./DatasetResetValidationSchema.json" -import { ErrorObject } from "../../types/ResponseModel"; import { ResponseHandler } from "../../helpers/ResponseHandler"; -import { DatasetStatus, DatasetType, HealthStatus } from "../../types/DatasetModels"; -import { Dataset } from "../../models/Dataset"; -import logger from "../../logger"; -import { getDruidIndexers, getFlinkHealthStatus, restartDruidIndexers } from "../../services/DatasetHealthService"; -import { Datasource } from "../../models/Datasource"; +import { HealthStatus } from "../../types/DatasetModels"; +import { getDruidIndexers, getFlinkHealthStatus, restartDruidIndexers } from "../../services/DatasetHealthService"; import { restartPipeline } from "../DatasetStatusTransition/DatasetStatusTransition"; +import { obsrvError } from "../../types/ObsrvError"; +import { datasetService } from "../../services/DatasetService"; +import httpStatus from "http-status"; export const apiId = "api.dataset.reset"; -export const errorCode = "DATASET_RESET_FAILURE" +const validateRequest = async (req: Request) => { + const isRequestValid: Record = schemaValidation(req.body, DatasetResetRequestSchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASET_INVALID_INPUT", isRequestValid.message, "BAD_REQUEST", 400) + } + const datasetId = _.get(req, ["params", "datasetId"]) + const isDataSetExists = await datasetService.checkDatasetExists(datasetId); + if (!isDataSetExists) { + throw obsrvError(datasetId, "DATASET_NOT_FOUND", `Dataset not exists with id:${datasetId}`, httpStatus[httpStatus.NOT_FOUND], 404) + } +} const datasetReset = async (req: Request, res: Response) => { - const resmsgid = _.get(res, "resmsgid"); - const category = req.body?.request?.category; - - const datasetId = _.get(req, "params.datasetId") - const msgid = _.get(req, ["body", "params", "msgid"]); - try { - const isRequestValid: Record = schemaValidation(req.body, DatasetResetRequestSchema) - if (!isRequestValid.isValid) { - const code = "DATASET_RESET_INPUT_INVALID" - logger.error({ code, apiId, msgid, category, resmsgid, message: isRequestValid.message }) - return ResponseHandler.errorResponse({ - code, - message: isRequestValid.message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); - } - const dataset = await getLiveDatasets(datasetId) - if(_.isEmpty(dataset)) { - const code = "DATASET_RESET_NO_DATASET" - const message = `There are no live dataset exists with given dataset_id: ${datasetId}` - logger.error({ code, apiId, msgid, category, resmsgid, message: message }) - return ResponseHandler.errorResponse({ - code, - message, - statusCode: 400, - errCode: "BAD_REQUEST" - } as ErrorObject, req, res); + const category = _.get(req, ["body", "request", "category"]); + const datasetId = _.get(req, ["params"," datasetId"]); + + await validateRequest(req); + if (category == "processing") { + const pipeLineStatus = await getFlinkHealthStatus() + if (pipeLineStatus == HealthStatus.UnHealthy) { + await restartPipeline({ "dataset": { "dataset_id": datasetId } }) } - logger.debug(apiId, msgid, resmsgid, "dataset", dataset, category) - const isMasterDataset = _.get(dataset, "[0].type") == DatasetType.master; - if(category == "processing") { - const pipeLineStatus = await getFlinkHealthStatus() - logger.debug({pipeLineStatus}) - if(pipeLineStatus == HealthStatus.UnHealthy){ - logger.debug("Restarting the pipeline") - await restartPipeline({"dataset": {"dataset_id": datasetId}}) - } - } else if(category == "query" && !isMasterDataset){ - const datasources = await getDataSources(datasetId) + } else if (category == "query") { + const datasources = await datasetService.findDatasources({"dataset_id": datasetId}) + if(!_.isEmpty(datasources)) { const unHealthySupervisors = await getDruidIndexers(datasources, HealthStatus.UnHealthy) const unHealthyDataSources = _.filter(unHealthySupervisors, (supervisor: any) => supervisor?.state == "SUSPENDED") - if(!_.isEmpty(unHealthyDataSources)){ + if (!_.isEmpty(unHealthyDataSources)) { await restartDruidIndexers(unHealthyDataSources) - } - } - - return ResponseHandler.successResponse(req, res, { - status: 200, data: { - "status": "Completed" } - }); - - } catch (error: any) { - logger.error({ error, apiId, code: errorCode, msgid, category, resmsgid }); - let errorMessage = error; - const statusCode = _.get(error, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: errorCode, message: "Failed to reset the dataset" } - } - ResponseHandler.errorResponse(errorMessage, req, res); + } } - - -} -const getLiveDatasets = async (ids: string): Promise> => { - return Dataset.findAll({ attributes: ["dataset_id", "status", "type"], where: { dataset_id: ids, status: DatasetStatus.Live }, raw: true }); -} - -const getDataSources = async (ids: string): Promise> => { - return Datasource.findAll({ attributes: ["dataset_id", "datasource"], where: { dataset_id: ids }, raw: true }); + return ResponseHandler.successResponse(req, res, { + status: 200, data: { + "status": "Completed" + } + }); } export default datasetReset; \ No newline at end of file From 1fbc072fa906934716053e1780493d9ce73dd45c Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Fri, 26 Jul 2024 11:21:14 +0530 Subject: [PATCH 064/235] #OBS-I2 typo fix --- api-service/src/services/DatasetHealthService.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/services/DatasetHealthService.ts b/api-service/src/services/DatasetHealthService.ts index b3fb094c..92672c47 100644 --- a/api-service/src/services/DatasetHealthService.ts +++ b/api-service/src/services/DatasetHealthService.ts @@ -104,7 +104,7 @@ export const getInfraHealth = async (isMasterDataset: boolean): Promise<{ compon export const getProcessingHealth = async (dataset: any): Promise<{ components: any, status: string }> => { const dataset_id = _.get(dataset, "dataset_id") const isMasterDataset = _.get(dataset, "type") == DatasetType.master; - const flink = await getKafkaHealthStatus() + const flink = await getFlinkHealthStatus() const { count, health } = await getEventsProcessedToday(dataset_id, isMasterDataset) const processingDefaultThreshold = await SystemConfig.getThresholds("processing") // eslint-disable-next-line prefer-const From 98248c0558dd5eff69f066c235b37c152f90e4e4 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 26 Jul 2024 14:30:01 +0530 Subject: [PATCH 065/235] #OBS-I116: fix: fix: Dataset copy check for dataset fix --- api-service/src/controllers/DatasetCopy/DatasetCopy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 8f86eeed..b33b614e 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -23,7 +23,7 @@ const fetchDataset = async (req: Request, newDatasetId: string) => { const isLive = _.get(req, "body.request.source.isLive"); let dataset = isLive ? await getLiveDatasetConfigs(datasetId) : await datasetService.getDraftDataset(datasetId) - if (dataset === null) { + if (!dataset) { throw obsrvError(datasetId, "DATASET_NOT_EXISTS", `Dataset ${datasetId} does not exists`, "NOT_FOUND", 404); } From 489c80114990f943ef708435cc948931be2ec54d Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Fri, 26 Jul 2024 18:13:15 +0530 Subject: [PATCH 066/235] #OBS-I21: dataset publish changes fixes --- .../templates/cronjob.yaml | 2 +- .../spark-connector-cron/templates/rbac.yaml | 48 ------------------- .../templates/serviceaccount.yaml | 11 ----- .../spark-connector-cron/values.yaml | 15 +----- .../src/command/connector_command.py | 2 - 5 files changed, 2 insertions(+), 76 deletions(-) delete mode 100644 command-service/helm-charts/spark-connector-cron/templates/rbac.yaml delete mode 100644 command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index f42637a4..07b70fb2 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -19,7 +19,7 @@ spec: {{- end }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 12 }} spec: - serviceAccountName: {{ include "base.serviceaccountname" . }} + serviceAccountName: {{ .Values.serviceAccount.name }} restartPolicy: {{ .Values.restartPolicy }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 12 }} diff --git a/command-service/helm-charts/spark-connector-cron/templates/rbac.yaml b/command-service/helm-charts/spark-connector-cron/templates/rbac.yaml deleted file mode 100644 index 7e291fd6..00000000 --- a/command-service/helm-charts/spark-connector-cron/templates/rbac.yaml +++ /dev/null @@ -1,48 +0,0 @@ -{{- if .Values.rbac.enabled -}} ---- -{{- if .Values.rbac.useClusterRole }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "common.names.fullname" . }} -rules: -{{- toYaml .Values.rbac.rules | nindent 2 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "common.names.fullname" . }} -subjects: -- kind: ServiceAccount - name: {{ include "base.serviceaccountname" . }} - namespace: {{ include "base.namespace" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "common.names.fullname" . }} -{{- else }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "common.names.fullname" . }} - namespace: {{ include "base.namespace" . }} -rules: -{{- toYaml .Values.rbac.rules | nindent 2 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "common.names.fullname" . }} - namespace: {{ include "base.namespace" . }} -subjects: -- kind: ServiceAccount - name: {{ include "base.serviceaccountname" . }} - namespace: {{ include "base.namespace" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "common.names.fullname" . }} -{{- end }} - -{{- end }} - diff --git a/command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml b/command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml deleted file mode 100644 index d26e8536..00000000 --- a/command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "base.serviceaccountname" . }} - namespace: {{ include "base.namespace" . }} - {{- if or .Values.serviceAccount.annotations .Values.commonAnnotations }} - {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceAccount.annotations .Values.commonAnnotations ) "context" . ) }} - annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} - {{- end }} -{{- end }} diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml index a781b613..1c4c2fce 100644 --- a/command-service/helm-charts/spark-connector-cron/values.yaml +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -92,20 +92,7 @@ configmap: mountPath: /config serviceAccount: - create: true - name: spark-cron-sa - annotations: {} - -rbac: - enabled: true - useClusterRole: true - rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["pods/exec"] - verbs: ["create"] + name: spark-connectors-sa serviceMonitor: enabled: false diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index e5f67970..2dbed28d 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -132,8 +132,6 @@ def _perform_spark_install(self, connector_instance): "--set", "main_file={}".format(connector_source["main_program"]), "--set", - "serviceAccount.create=false", - "--set", "cronSchedule={}".format(connector_instance.operations_config["schedule"]) ] From 63adc0e3b0248a2cc1c8e131a13f119177189d21 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 26 Jul 2024 23:46:39 +0530 Subject: [PATCH 067/235] #OBS-I116: fix: feat: Feedback fixes of removing set redis db --- .../controllers/DatasetCopy/DatasetCopyHelper.ts | 12 ------------ .../controllers/DatasetExport/DatasetExport.ts | 15 +-------------- 2 files changed, 1 insertion(+), 26 deletions(-) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts index 7de9c953..de4c8087 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -1,7 +1,6 @@ import * as _ from "lodash"; import { DatasetStatus } from "../../types/DatasetModels"; import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; -import { query } from "../../connections/databaseConnection"; import { config } from "../../configs/Config"; const version = defaultDatasetConfig.version; @@ -16,15 +15,4 @@ export const updateRecords = (datasetRecord: Record, newDatasetId: _.set(datasetRecord, 'version', version); _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) _.set(datasetRecord, "router_config", { topic: newDatasetId }) - - if (datasetRecord?.type === "master") { - _.set(datasetRecord, "dataset_config.cache_config.redis_db", updateMasterDatasetConfig(datasetRecord?.dataset_config)) - } -} - -const updateMasterDatasetConfig = async (datasetConfig: any) => { - let nextRedisDB = datasetConfig.redis_db; - const { results }: any = await query("SELECT nextval('redis_db_index')") - if (!_.isEmpty(results)) nextRedisDB = parseInt(_.get(results, "[0].nextval")) || 3; - return _.assign(datasetConfig, { "redis_db": nextRedisDB }) } diff --git a/api-service/src/controllers/DatasetExport/DatasetExport.ts b/api-service/src/controllers/DatasetExport/DatasetExport.ts index 23f54d41..3e118639 100644 --- a/api-service/src/controllers/DatasetExport/DatasetExport.ts +++ b/api-service/src/controllers/DatasetExport/DatasetExport.ts @@ -31,21 +31,8 @@ const validateDataset = async (req: Request) => { const datasetExport = async (req: Request, res: Response) => { const datasetRecord = await validateDataset(req) - const updateDataset = transformConnectorsConfig(datasetRecord) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: updateDataset }); + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: datasetRecord }); } -const transformConnectorsConfig = (dataset: Record) => { - const connectorConfig = _.get(dataset, "connectors_config") - if (!_.isEmpty(connectorConfig)) { - const updatedConnectorConfig = _.map(connectorConfig, config => { - const { id, connector_id, operations_config, version } = config - return { id, connector_id, connector_config: {}, operations_config, version } - }) - return { ...dataset, connectors_config: updatedConnectorConfig } - } - return dataset -} - export default datasetExport; \ No newline at end of file From 2825ebcf0f9a003da429dfc02f57fd7855cbc733 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 29 Jul 2024 13:19:21 +0530 Subject: [PATCH 068/235] #OBS-I138: added cors to app --- api-service/package.json | 8 +++++--- api-service/src/app.ts | 4 +++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/api-service/package.json b/api-service/package.json index f760df95..ff5a5831 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -5,7 +5,7 @@ "main": "dist/app.js", "scripts": { "start": "ts-node ./src/app.ts", - "test": "source .env.test && nyc mocha ./src/v1/test/*.spec.ts --exit && nyc mocha ./src/v2/tests/**/*.spec.ts --exit", + "test": "source .env.test && nyc mocha ./src/v2/tests/**/GenerateSignedURL.spec.ts --exit", "actions:test": "nyc mocha ./src/v1/test/*.spec.ts --exit", "actions:test:v2": "nyc mocha ./src/v2/tests/**/*.spec.ts --exit", "build": "rm -rf dist && tsc --declaration -P . && cp package.json ./dist/package.json", @@ -16,13 +16,13 @@ "author": "Obsrv", "license": "MIT", "dependencies": { + "@aws-sdk/abort-controller": "^3.374.0", "@aws-sdk/client-s3": "^3.540.0", "@aws-sdk/credential-providers": "^3.309.0", - "@aws-sdk/lib-storage": "^3.182.0", + "@aws-sdk/lib-storage": "^3.609.0", "@aws-sdk/s3-request-presigner": "^3.540.0", "@azure/storage-blob": "^12.17.0", "@google-cloud/storage": "^7.9.0", - "@jsonhero/schema-infer": "^0.1.5", "@project-sunbird/logger": "^0.0.9", "ajv": "^8.11.2", "ajv-formats": "^2.1.1", @@ -30,6 +30,7 @@ "axios": "^1.6.0", "body-parser": "^1.20.2", "compression": "^1.7.4", + "cors": "^2.8.5", "dateformat": "2.0.0", "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", @@ -44,6 +45,7 @@ "moment": "^2.29.4", "multiparty": "4.2.1", "node-sql-parser": "^5.1.0", + "obsrv-config-service-ext": "file:obsrv-config-service-ext-1.0.0.tgz", "pg": "^8.11.3", "pg-hstore": "^2.3.4", "prom-client": "^14.2.0", diff --git a/api-service/src/app.ts b/api-service/src/app.ts index d135e7bc..dd184e2b 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -8,9 +8,11 @@ import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; import { alertsRouter } from "./routes/AlertsRouter"; +// import cors from "cors"; +const cors = require("cors"); const app: Application = express(); - +app.use(cors()); app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); From 8b9963f88c45fd81e5e8e7b8e7e8d442ad79e6ed Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 29 Jul 2024 18:07:21 +0530 Subject: [PATCH 069/235] #OBS-I116: fix: feat: Dataset import api implementation --- .../DatasetImport/DatasetImport.ts | 51 ++ .../DatasetImport/DatasetImportHelper.ts | 150 ++++++ .../RequestValidationSchema.json | 453 ++++++++++++++++++ api-service/src/routes/Router.ts | 2 + 4 files changed, 656 insertions(+) create mode 100644 api-service/src/controllers/DatasetImport/DatasetImport.ts create mode 100644 api-service/src/controllers/DatasetImport/DatasetImportHelper.ts create mode 100644 api-service/src/controllers/DatasetImport/RequestValidationSchema.json diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts new file mode 100644 index 00000000..dfd32813 --- /dev/null +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -0,0 +1,51 @@ +import { Request, Response } from "express"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import _ from "lodash"; +import { datasetService } from "../../services/DatasetService"; +import { datasetImportValidation, migrateExportedDatasetV1 } from "./DatasetImportHelper"; +import { obsrvError } from "../../types/ObsrvError"; + +const datasetImport = async (req: Request, res: Response) => { + + const requestBody = req.body + let datasetPayload = requestBody.request; + if (_.get(datasetPayload, "api_version") !== "v2") { + const migratedConfigs = migrateExportedDatasetV1(datasetPayload) + datasetPayload = migratedConfigs; + } + const dataset_id = _.get(datasetPayload, "dataset_id") + const { updatedDataset, ignoredFields } =await datasetImportValidation({ ...requestBody, "request": datasetPayload }) + const { successMsg, partialIgnored } = getResponseData(ignoredFields) + const dataset = await datasetService.createDraftDataset(updatedDataset).catch(err => { + if (err?.name === 'SequelizeUniqueConstraintError') { + throw obsrvError(dataset_id, "DATASET_ALREADY_EXISTS", `Dataset with id ${dataset_id} already exists to import`, "BAD_REQUEST", 400); + } + throw obsrvError("", "DATASET_IMPORT_FAILURE", `Failed to import dataset`, "INTERNAL_SERVER_ERROR", 500); + }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); +} + +const getResponseData = (ignoredConfigs: Record) => { + const { ignoredConnectors, ignoredTransformations, ignoredDenorms } = ignoredConfigs; + let successMsg = "Dataset is imported successfully"; + let partialIgnored: Record = {}; + + if (ignoredConnectors.length || ignoredTransformations.length || ignoredDenorms.length) { + successMsg = "Dataset is partially imported"; + + if (ignoredTransformations.length) { + partialIgnored.transformations = ignoredTransformations; + } + if (ignoredConnectors.length) { + partialIgnored.connectors = ignoredConnectors; + } + if (ignoredDenorms.length) { + partialIgnored.denorm_fields = ignoredDenorms; + } + } + + return { successMsg, partialIgnored }; +} + +export default datasetImport; \ No newline at end of file diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts new file mode 100644 index 00000000..beb45c73 --- /dev/null +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -0,0 +1,150 @@ +import Ajv from "ajv"; +import _ from "lodash"; +import { obsrvError } from "../../types/ObsrvError"; +import ValidationSchema from "./RequestValidationSchema.json" +import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; +import { schemaValidation } from "../../services/ValidationService"; +import { DatasetStatus, DatasetType } from "../../types/DatasetModels"; +import { datasetService } from "../../services/DatasetService"; +const validator = new Ajv(); + +const reqBodySchema = ValidationSchema.request_body +const transformationSchema = ValidationSchema.transformations_config +const connectorSchema = ValidationSchema.connectors_config +const denormSchema = ValidationSchema.denorm_config + +const validateConfigs = (schema: any, configs: any[], validKey: string): { valid: any[], ignored: any[] } => { + const validConfigs: any[] = []; + const ignoredConfigs: any[] = []; + + for (const config of configs) { + if (validator.validate(schema, config)) { + validConfigs.push(config); + } else { + const error: any = validator.errors; + const errorMessage = error[0]?.schemaPath?.replace("/", "") + " " + error[0]?.message || "Invalid Request Body"; + ignoredConfigs.push({ config, reason: errorMessage }); + } + } + + return { valid: validConfigs, ignored: ignoredConfigs }; +}; + +export const datasetImportValidation = async (payload: Record): Promise> => { + const isRequestValid: Record = schemaValidation(payload, reqBodySchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "DATASET_IMPORT_INVALID_CONFIGS", isRequestValid.message, "BAD_REQUEST", 400) + } + + let datasetConfig = payload.request; + + const connectors = _.get(datasetConfig, "connectors_config", []); + const transformations = _.get(datasetConfig, "transformations_config", []); + const denormConfig = _.get(datasetConfig, "denorm_config", { denorm_fields: [] }); + const { validDenorms, invalidDenorms } = await validateDenorms(denormConfig) + + const { valid: resultantConnectors, ignored: ignoredConnectors } = validateConfigs(connectorSchema, connectors, 'connector_id'); + const { valid: resultantTransformations, ignored: ignoredTransformations } = validateConfigs(transformationSchema, transformations, 'field_key'); + const { valid: resultantDenorms, ignored: ignoredDenorms } = validateConfigs(denormSchema, validDenorms, 'denorm_out_field'); + + datasetConfig["connectors_config"] = resultantConnectors; + datasetConfig["transformations_config"] = resultantTransformations; + datasetConfig["denorm_config"] = { ...denormConfig, denorm_fields: resultantDenorms }; + datasetConfig["router_config"] = { topic: datasetConfig.id } + datasetConfig["version_key"] = Date.now().toString() + + const defaults = _.cloneDeep(defaultDatasetConfig); + const resultantDataset = _.merge(defaults, datasetConfig); + + return { + updatedDataset: _.omit(resultantDataset, ["created_date", "updated_date", "published_date", "status"]), + ignoredFields: { ignoredConnectors, ignoredTransformations, ignoredDenorms: [...ignoredDenorms, ...invalidDenorms] } + }; +}; + +const validateDenorms = async (denormConfig: Record): Promise> => { + const invalidDenorms: any[] = []; + const validDenorms: any[] = []; + + if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id", "dataset_id", "status", "dataset_config", "api_version"]); + + for (const field of denormConfig.denorm_fields) { + const { redis_db, dataset_id, denorm_out_field, denorm_key } = field; + let masterDataset; + + if (dataset_id) { + masterDataset = _.find(masterDatasets, dataset => _.get(dataset, "dataset_id") === dataset_id); + } else if (redis_db) { + masterDataset = _.find(masterDatasets, dataset => _.get(dataset, "dataset_config.cache_config.redis_db") === redis_db); + } + + const denormFields = { denorm_key, denorm_out_field, dataset_id: dataset_id || _.get(masterDataset, "dataset_id") } + if (masterDataset) { + validDenorms.push(denormFields); + } else { + invalidDenorms.push({ config: denormFields, reason: `Master dataset does not exist` }); + } + } + } + + return { validDenorms, invalidDenorms }; +}; + +export const migrateExportedDatasetV1 = (datasetPayload: Record) => { + + const { dataset_id, timestamp_key = "", data_key = "", type: datasetType } = _.get(datasetPayload, "data.metadata") + const type = datasetType === "master-dataset" ? DatasetType.master : DatasetType.event + + let dataset: Record = { + dataset_id, id: dataset_id, name: dataset_id, type, + version_key: Date.now().toString(), + api_version: "v2", + }; + + const { validation, dedup, batch } = _.get(datasetPayload, "data.config") + dataset["data_schema"] = _.get(datasetPayload, "data.data_schema") + dataset["dedup_config"] = { ..._.omit(dedup, "enabled"), drop_duplicates: _.get(dedup, "enabled") }; + dataset["router_config"] = { topic: dataset_id }; + dataset["validation_config"] = { ..._.omit(validation, "enabled"), validate: _.get(validation, "enabled") }; + + const { drop_duplicates, dedup_key, dedup_period, extraction_key, enabled: is_batch_event } = batch + dataset["extraction_config"] = { is_batch_event, extraction_key, dedup_config: { drop_duplicates, dedup_key, dedup_period } } + + const { redis_db, redis_db_host, redis_db_port } = defaultDatasetConfig.dataset_config.cache_config; + dataset["dataset_config"] = { + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (type === DatasetType.master) }, + keys_config: { data_key, timestamp_key }, + cache_config: { redis_db_host, redis_db_port, redis_db } + } + + dataset["denorm_config"] = { + denorm_fields: _.map(_.get(datasetPayload, "data.denorm"), configs => { + const { master_dataset_id, denorm_key, out } = configs; + return { denorm_key, denorm_out_field: out, dataset_id: master_dataset_id } + }) + } + + dataset["transformations_config"] = _.map(_.get(datasetPayload, "data.transformations", []), (config) => { + const { type, key, expr, mode, dataType = "string", section } = config + return { + field_key: key, + transformation_function: { + type, expr, + datatype: dataType, + category: datasetService.getTransformationCategory(section) + }, mode + } + }) + + dataset["connectors_config"] = _.map(_.get(datasetPayload, "env_variables.input_sources", []), (config) => { + const { id, type, ...rest } = config + return { + id, connector_id: type, + connector_config: rest, + version: "v1" + } + }) + + return dataset; +} \ No newline at end of file diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchema.json b/api-service/src/controllers/DatasetImport/RequestValidationSchema.json new file mode 100644 index 00000000..8efca393 --- /dev/null +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchema.json @@ -0,0 +1,453 @@ +{ + "request_body": { + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasets.import"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "enum": ["event", "transaction", "master"] + }, + "version": { + "type": "integer" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "validation_config": { + "type": "object", + "properties": { + "validate": { + "type": "boolean", + "default": true + }, + "mode": { + "type": "string", + "enum": ["Strict", "IgnoreNewFields"], + "default": "Strict" + } + }, + "required": ["validate"], + "additionalProperties": false, + "if": { + "properties": { + "validate": { + "const": true + } + } + }, + "then": { + "required": ["mode"] + } + }, + "extraction_config": { + "type": "object", + "properties": { + "is_batch_event": { + "type": "boolean", + "default": true + }, + "extraction_key": { + "type": "string", + "default": "events" + }, + "dedup_config": { + "type": "object", + "properties": { + "drop_duplicates": { + "type": "boolean", + "default": false + }, + "dedup_key": { + "type": "string" + }, + "dedup_period": { + "type": "integer" + } + }, + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], + "additionalProperties": false + } + }, + "additionalProperties": false, + "required": ["is_batch_event"], + "if": { + "properties": { + "is_batch_event": { + "const": true + } + } + }, + "then": { + "properties": { + "extraction_key": { + "minLength": 1 + } + }, + "required": ["extraction_key", "dedup_config"] + } + }, + "dedup_config": { + "type": "object", + "properties": { + "drop_duplicates": { + "type": "boolean", + "default": true + }, + "dedup_key": { + "type": "string" + }, + "dedup_period": { + "type": "integer" + } + }, + "if": { + "properties": { + "drop_duplicates": { + "const": true + } + } + }, + "then": { + "properties": { + "dedup_key": { + "minLength": 1 + } + }, + "required": ["dedup_key"] + }, + "required": ["drop_duplicates"], + "additionalProperties": false + }, + "data_schema": { + "type": "object", + "properties": { + "$schema": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "minLength": 1 + }, + "properties": { + "type": "object", + "minProperties": 1 + }, + "additionalProperties": { + "type": "boolean", + "default": true + }, + "required": { + "type": "array" + } + }, + "required": ["$schema", "type", "properties"], + "additionalProperties": false + }, + "dataset_config": { + "type": "object", + "properties": { + "indexing_config": { + "type": "object", + "properties": { + "olap_store_enabled": { + "type": "boolean", + "default": false + }, + "lakehouse_enabled": { + "type": "boolean", + "default": true + }, + "cache_enabled": { + "type": "boolean", + "default": false + } + }, + "additionalProperties": false + }, + "keys_config": { + "type": "object", + "properties": { + "data_key": { + "type": "string" + }, + "partition_key": { + "type": "string" + }, + "timestamp_key": { + "type": "string" + }, + "timestamp_format": { + "type": "string", + "minLength": 1 + } + }, + "additionalProperties": false + }, + "cache_config": { + "type": "object", + "properties": { + "redis_db_host": { + "type": "string", + "minLength": 1 + }, + "redis_db_port": { + "type": "integer" + }, + "redis_db": { + "type": "integer" + } + }, + "required": ["redis_db_host", "redis_db_port"], + "additionalProperties": false + }, + "file_upload_path": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "required": ["indexing_config", "keys_config"], + "additionalProperties": false + }, + "transformations_config": { + "type": "array" + }, + "connectors_config": { + "type": "array" + }, + "denorm_config": { + "type": "object", + "properties": { + "redis_db_host": { + "type": "string" + }, + "redis_db_port": { + "type": "integer" + }, + "denorm_fields": { + "type": "array" + } + }, + "required": ["denorm_fields"], + "additionalProperties": false + }, + "tags": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "router_config": { + "type": "object", + "properties": { + "topic": { + "type": "string", + "minLength": 1 + } + }, + "required": ["topic"], + "additionalProperties": false + }, + "sample_data": { + "type": "object" + }, + "entry_topic": { + "type": "string", + "minLength": 1 + }, + "data_version": { + "type": "integer" + }, + "api_version": { + "type": "string", + "minLength": 1 + }, + "version_key": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string" + }, + "created_date": { + "type": "string" + }, + "updated_date": { + "type": "string" + }, + "created_by": { + "type": "string" + }, + "updated_by": { + "type": "string" + } + }, + "required": [ + "id", + "dataset_id", + "name", + "type", + "data_schema", + "dataset_config" + ], + "additionalProperties": false + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false + }, + "transformations_config": { + "type": "object", + "properties": { + "field_key": { + "type": "string", + "minLength": 1 + }, + "transformation_function": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string" + }, + "condition": { + "type": "object", + "properties": { + "type": { + "type": "string", + "minLength": 1 + }, + "expr": { + "type": "string", + "minLength": 1 + } + }, + "required": ["type", "expr"], + "additionalProperties": false + }, + "datatype": { + "type": "string" + }, + "category": { + "type": "string", + "enum": ["pii", "transform", "derived"] + } + }, + "required": ["type", "expr"], + "additionalProperties": false + }, + "mode": { + "type": "string", + "enum": ["Strict", "Lenient"] + } + }, + "additionalProperties": false, + "required": ["field_key", "transformation_function", "mode"] + }, + "connectors_config": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "connector_id": { + "type": "string", + "minLength": 1 + }, + "connector_config": { + "oneOf": [{ "type": "string" }, { "type": "object" }] + }, + "operations_config": { + "type": "object" + }, + "version": { + "type": "string" + } + }, + "additionalProperties": false, + "required": ["id", "connector_id", "connector_config"] + }, + "denorm_config": { + "type": "object", + "properties": { + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "denorm_key": { + "type": "string", + "minLength": 1 + }, + "denorm_out_field": { + "type": "string", + "minLength": 1 + }, + "jsonata_expr": { + "type": "string", + "minLength": 1 + } + }, + "oneOf": [ + { + "required": ["dataset_id", "denorm_out_field", "denorm_key"] + }, + { + "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] + } + ] + } +} diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index fda6de43..c9509ab2 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -24,6 +24,7 @@ import DataSchemaGenerator from "../controllers/GenerateDataSchema/GenerateDataS import datasetReset from "../controllers/DatasetReset/DatasetReset"; import DatasetExport from "../controllers/DatasetExport/DatasetExport"; import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; +import DatasetImport from "../controllers/DatasetImport/DatasetImport"; export const router = express.Router(); @@ -48,6 +49,7 @@ router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.res router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), DatasetCopy); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file From bf5058343462c0e9ef91b9c1b01c0e9a99f9fd45 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 29 Jul 2024 18:18:45 +0530 Subject: [PATCH 070/235] #OBS-I116: fix: fix: Dataset service fix --- .../DatasetImport/DatasetImportHelper.ts | 14 +++++++------- api-service/src/services/DatasetService.ts | 13 ++++++++----- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts index beb45c73..48bc47a2 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -13,7 +13,7 @@ const transformationSchema = ValidationSchema.transformations_config const connectorSchema = ValidationSchema.connectors_config const denormSchema = ValidationSchema.denorm_config -const validateConfigs = (schema: any, configs: any[], validKey: string): { valid: any[], ignored: any[] } => { +const validateConfigs = (schema: any, configs: any[]): { valid: any[], ignored: any[] } => { const validConfigs: any[] = []; const ignoredConfigs: any[] = []; @@ -23,7 +23,7 @@ const validateConfigs = (schema: any, configs: any[], validKey: string): { valid } else { const error: any = validator.errors; const errorMessage = error[0]?.schemaPath?.replace("/", "") + " " + error[0]?.message || "Invalid Request Body"; - ignoredConfigs.push({ config, reason: errorMessage }); + ignoredConfigs.push({ config, details: errorMessage }); } } @@ -43,9 +43,9 @@ export const datasetImportValidation = async (payload: Record): Pro const denormConfig = _.get(datasetConfig, "denorm_config", { denorm_fields: [] }); const { validDenorms, invalidDenorms } = await validateDenorms(denormConfig) - const { valid: resultantConnectors, ignored: ignoredConnectors } = validateConfigs(connectorSchema, connectors, 'connector_id'); - const { valid: resultantTransformations, ignored: ignoredTransformations } = validateConfigs(transformationSchema, transformations, 'field_key'); - const { valid: resultantDenorms, ignored: ignoredDenorms } = validateConfigs(denormSchema, validDenorms, 'denorm_out_field'); + const { valid: resultantConnectors, ignored: ignoredConnectors } = validateConfigs(connectorSchema, connectors); + const { valid: resultantTransformations, ignored: ignoredTransformations } = validateConfigs(transformationSchema, transformations); + const { valid: resultantDenorms, ignored: ignoredDenorms } = validateConfigs(denormSchema, validDenorms); datasetConfig["connectors_config"] = resultantConnectors; datasetConfig["transformations_config"] = resultantTransformations; @@ -67,7 +67,7 @@ const validateDenorms = async (denormConfig: Record): Promise): Promise { return { field_key: _.get(config, ["field_key"]), - transformation_function: _.get(config, ["transformation_function"]), - mode: _.get(config, ["mode"]), - datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + transformation_function: { + ..._.get(config, ["transformation_function"]), + datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", + category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + }, + mode: _.get(config, ["mode"]) } }) const connectors = await this.getDraftConnectors(dataset_id, ["id", "connector_type", "connector_config"]); @@ -138,10 +140,11 @@ class DatasetService { version: "v1" } }) + draftDataset["status"] = DatasetStatus.Draft return draftDataset; } - private getTransformationCategory = (section: string):string => { + getTransformationCategory = (section: string):string => { switch(section) { case "pii": From d3a12c3222a09aa96c5f239ad211d06b50d6ddb4 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 29 Jul 2024 18:27:33 +0530 Subject: [PATCH 071/235] #OBS-I116: fix: repeated Validation method removal --- .../src/controllers/DatasetImport/DatasetImportHelper.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts index 48bc47a2..8a94c8fa 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -1,4 +1,3 @@ -import Ajv from "ajv"; import _ from "lodash"; import { obsrvError } from "../../types/ObsrvError"; import ValidationSchema from "./RequestValidationSchema.json" @@ -6,7 +5,6 @@ import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; import { schemaValidation } from "../../services/ValidationService"; import { DatasetStatus, DatasetType } from "../../types/DatasetModels"; import { datasetService } from "../../services/DatasetService"; -const validator = new Ajv(); const reqBodySchema = ValidationSchema.request_body const transformationSchema = ValidationSchema.transformations_config @@ -18,12 +16,11 @@ const validateConfigs = (schema: any, configs: any[]): { valid: any[], ignored: const ignoredConfigs: any[] = []; for (const config of configs) { - if (validator.validate(schema, config)) { + const isConfig = schemaValidation(config, schema) + if (isConfig.isValid) { validConfigs.push(config); } else { - const error: any = validator.errors; - const errorMessage = error[0]?.schemaPath?.replace("/", "") + " " + error[0]?.message || "Invalid Request Body"; - ignoredConfigs.push({ config, details: errorMessage }); + ignoredConfigs.push({ config, details: isConfig.message }); } } From b5a03f0a0c68144b5e983d5718bc611160bf82ad Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 29 Jul 2024 18:32:51 +0530 Subject: [PATCH 072/235] #OBS-I116: fix: unused code --- api-service/src/services/DatasetService.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index f869e447..bb1d3efc 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -140,7 +140,6 @@ class DatasetService { version: "v1" } }) - draftDataset["status"] = DatasetStatus.Draft return draftDataset; } From dba36d02c856fa0789dedd6d80ccdf4605e9f339 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 30 Jul 2024 11:21:51 +0530 Subject: [PATCH 073/235] connector list --- .../ConnectorsList/ConnectorsList.ts | 43 +++++++++++++++ .../ConnectorsListValidationSchema.json | 55 +++++++++++++++++++ api-service/src/routes/Router.ts | 2 + api-service/src/services/ConnectorService.ts | 11 ++++ 4 files changed, 111 insertions(+) create mode 100644 api-service/src/controllers/ConnectorsList/ConnectorsList.ts create mode 100644 api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json create mode 100644 api-service/src/services/ConnectorService.ts diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts new file mode 100644 index 00000000..022eaf3e --- /dev/null +++ b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts @@ -0,0 +1,43 @@ +import { Request, Response } from "express"; +import ConnectorListSchema from "./ConnectorsListValidationSchema.json"; +import { obsrvError } from "../../types/ObsrvError"; +import { schemaValidation } from "../../services/ValidationService"; +import _ from "lodash"; +import logger from "../../logger"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import { connectorService } from "../../services/ConnectorService"; +import { Op } from "sequelize"; + +export const apiId = "api.connectors.list"; +export const errorCode = "CONNECTORS_LIST_FAILURE"; +const defaultFeilds = ["id","connector_id","name", "type", "category", "version", "description", "technology", "runtime", "licence", "owner", "iconurl", "status", "created_by", "updated_by", "created_date", "updated_date"]; + +const validateRequest = (req: Request)=>{ + const isRequestValid: Record = schemaValidation(req.body, ConnectorListSchema) + if (!isRequestValid.isValid) { + throw obsrvError("", "CONNECTORS_LIST_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) + } +} + +const connectorsList = async (req: Request, res: Response) => { + validateRequest(req); + + const connectorBody = req.body.request; + const connectorList = await listConnectors(connectorBody) + const responseData = { data: connectorList, count: _.size(connectorList) } + logger.info({req: req.body, resmsgid: _.get(res, "resmsgid"), message: `Connectors are listed successfully with a connectors count (${_.size(connectorList)})` }) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); +} + +const listConnectors = async (request: Record): Promise> => { + const { filters = {} } = request || {}; + const connectorStatus = _.get(filters, "status"); + const connectorCategory = _.get(filters,"category"); + const filterOptions ={ ...!_.isEmpty(connectorStatus)? { status: { [Op.in]: connectorStatus } } : {}, + ...!_.isEmpty(connectorCategory)? {category: {[Op.in]: connectorCategory}}: {} } + const filteredconnectorList = await connectorService.findConnectors(filterOptions,defaultFeilds); + return filteredconnectorList ; +} + +export default connectorsList; \ No newline at end of file diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json b/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json new file mode 100644 index 00000000..65afbdeb --- /dev/null +++ b/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json @@ -0,0 +1,55 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.connectors.list"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "filters": { + "type": "object", + "properties": { + "status": { + "type": "array", + "items": { + "type": "string", + "enum": ["Draft", "InValidation", "Live", "Retired"] + }, + "minItems": 1 + }, + "category": { + "type": "array", + "items": { + "type": "string", + "enum": ["Database", "File"] + }, + "minItems": 1 + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false +} \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index fda6de43..4ba84153 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -24,6 +24,7 @@ import DataSchemaGenerator from "../controllers/GenerateDataSchema/GenerateDataS import datasetReset from "../controllers/DatasetReset/DatasetReset"; import DatasetExport from "../controllers/DatasetExport/DatasetExport"; import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; +import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; export const router = express.Router(); @@ -48,6 +49,7 @@ router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.res router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), ConnectorsList); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file diff --git a/api-service/src/services/ConnectorService.ts b/api-service/src/services/ConnectorService.ts new file mode 100644 index 00000000..e6fb2034 --- /dev/null +++ b/api-service/src/services/ConnectorService.ts @@ -0,0 +1,11 @@ +import { ConnectorRegistry } from "../models/ConnectorRegistry"; + +class ConnectorService{ + + findConnectors = async (where?: Record, attributes?: string[]): Promise => { + return ConnectorRegistry.findAll({where, attributes, raw: true}); + } + +} + +export const connectorService = new ConnectorService(); \ No newline at end of file From 81fc51143ac2fd3a87996f3afdd7943e2c652f36 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 30 Jul 2024 12:33:34 +0530 Subject: [PATCH 074/235] #OBS-I142: connector list api --- .../controllers/ConnectorsList/ConnectorsList.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts index 022eaf3e..61bee8bf 100644 --- a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts +++ b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts @@ -7,10 +7,7 @@ import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; import { connectorService } from "../../services/ConnectorService"; -import { Op } from "sequelize"; -export const apiId = "api.connectors.list"; -export const errorCode = "CONNECTORS_LIST_FAILURE"; const defaultFeilds = ["id","connector_id","name", "type", "category", "version", "description", "technology", "runtime", "licence", "owner", "iconurl", "status", "created_by", "updated_by", "created_date", "updated_date"]; const validateRequest = (req: Request)=>{ @@ -34,8 +31,14 @@ const listConnectors = async (request: Record): Promise Date: Tue, 30 Jul 2024 12:56:24 +0530 Subject: [PATCH 075/235] #OBS-I116: fix: schema validation check for v1 exported dataset. --- .../DatasetImport/DatasetImport.ts | 28 ++- .../DatasetImport/DatasetImportHelper.ts | 18 +- .../RequestValidationSchemaV1.json | 235 ++++++++++++++++++ ...ma.json => RequestValidationSchemaV2.json} | 0 4 files changed, 269 insertions(+), 12 deletions(-) create mode 100644 api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json rename api-service/src/controllers/DatasetImport/{RequestValidationSchema.json => RequestValidationSchemaV2.json} (100%) diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index dfd32813..565550ee 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -8,22 +8,38 @@ import { obsrvError } from "../../types/ObsrvError"; const datasetImport = async (req: Request, res: Response) => { + const { overwrite = "true" } = req.query; const requestBody = req.body let datasetPayload = requestBody.request; if (_.get(datasetPayload, "api_version") !== "v2") { - const migratedConfigs = migrateExportedDatasetV1(datasetPayload) + const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } - const dataset_id = _.get(datasetPayload, "dataset_id") - const { updatedDataset, ignoredFields } =await datasetImportValidation({ ...requestBody, "request": datasetPayload }) + const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) - const dataset = await datasetService.createDraftDataset(updatedDataset).catch(err => { + + const dataset = overwrite == "true" ? await overWriteDataset(updatedDataset) : await createNewDataset(updatedDataset) + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); +} + +const createNewDataset = async (dataset: Record) => { + return datasetService.createDraftDataset(dataset).catch(err => { if (err?.name === 'SequelizeUniqueConstraintError') { - throw obsrvError(dataset_id, "DATASET_ALREADY_EXISTS", `Dataset with id ${dataset_id} already exists to import`, "BAD_REQUEST", 400); + const dataset_id = _.get(dataset, "dataset_id") + throw obsrvError(dataset.dataset_id, "DATASET_ALREADY_EXISTS", `Dataset with id : ${dataset_id} already exists to import`, "BAD_REQUEST", 400); } throw obsrvError("", "DATASET_IMPORT_FAILURE", `Failed to import dataset`, "INTERNAL_SERVER_ERROR", 500); }) - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); +} + +const overWriteDataset = async (dataset: Record) => { + const dataset_id = _.get(dataset, "dataset_id") + const draftDataset = await datasetService.getDraftDataset(dataset_id, ["id"]) + if (!draftDataset) { + throw obsrvError(dataset.dataset_id, "DATASET_NOT_FOUND", `Dataset with dataset_id: ${dataset_id} not found to overwrite`, "NOT_FOUND", 404) + } + const response = await datasetService.updateDraftDataset(dataset) + return _.omit(response, ["message"]) } const getResponseData = (ignoredConfigs: Record) => { diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts index 8a94c8fa..9388ab21 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -1,15 +1,16 @@ import _ from "lodash"; import { obsrvError } from "../../types/ObsrvError"; -import ValidationSchema from "./RequestValidationSchema.json" +import ValidationSchemaV2 from "./RequestValidationSchemaV2.json" +import ValidationSchemaV1 from "./RequestValidationSchemaV1.json" import { defaultDatasetConfig } from "../../configs/DatasetConfigDefault"; import { schemaValidation } from "../../services/ValidationService"; import { DatasetStatus, DatasetType } from "../../types/DatasetModels"; import { datasetService } from "../../services/DatasetService"; -const reqBodySchema = ValidationSchema.request_body -const transformationSchema = ValidationSchema.transformations_config -const connectorSchema = ValidationSchema.connectors_config -const denormSchema = ValidationSchema.denorm_config +const reqBodySchema = ValidationSchemaV2.request_body +const transformationSchema = ValidationSchemaV2.transformations_config +const connectorSchema = ValidationSchemaV2.connectors_config +const denormSchema = ValidationSchemaV2.denorm_config const validateConfigs = (schema: any, configs: any[]): { valid: any[], ignored: any[] } => { const validConfigs: any[] = []; @@ -88,8 +89,13 @@ const validateDenorms = async (denormConfig: Record): Promise) => { +export const migrateExportedDatasetV1 = (requestPayload: Record) => { + const v1Config = schemaValidation(requestPayload, ValidationSchemaV1); + if (!v1Config.isValid) { + throw obsrvError("", "DATASET_V1_CONFIGS_INVALID", v1Config.message, "BAD_REQUEST", 400) + } + const datasetPayload = requestPayload.request const { dataset_id, timestamp_key = "", data_key = "", type: datasetType } = _.get(datasetPayload, "data.metadata") const type = datasetType === "master-dataset" ? DatasetType.master : DatasetType.event diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json new file mode 100644 index 00000000..21a4b795 --- /dev/null +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json @@ -0,0 +1,235 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string", + "enum": ["api.datasets.import"] + }, + "ver": { + "type": "string" + }, + "ts": { + "type": "string" + }, + "params": { + "type": "object", + "properties": { + "msgid": { + "type": "string" + } + }, + "required": ["msgid"], + "additionalProperties": false + }, + "request": { + "type": "object", + "properties": { + "version": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "env_variables": { + "type": "object", + "properties": { + "input_sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string" + } + }, + "required": ["id", "type"] + } + } + }, + "required": ["input_sources"] + }, + "data": { + "type": "object", + "properties": { + "metadata": { + "type": "object", + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "dataset_id": { + "type": "string", + "minLength": 1 + }, + "type": { + "type": "string", + "enum": ["dataset", "master-dataset"] + }, + "timestamp_key": { + "type": "string", + "minLength": 1 + }, + "data_key": { + "type": "string", + "minLength": 1 + } + }, + "required": ["id", "dataset_id", "type"] + }, + "data_schema": { + "type": "object", + "properties": { + "$schema": { + "type": "string" + }, + "type": { + "type": "string" + }, + "properties": { + "type": "object" + }, + "additionalProperties": { + "type": "boolean" + } + }, + "required": ["$schema", "type", "properties"] + }, + "config": { + "type": "object", + "properties": { + "batch": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "extraction_key": { + "type": "string" + }, + "drop_duplicates": { + "type": "boolean" + }, + "dedup_period": { + "type": "integer" + }, + "dedup_key": { + "type": "string" + } + }, + "required": [ + "enabled", + "id", + "extraction_key", + "drop_duplicates", + "dedup_key" + ] + }, + "dedup": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "dedup_key": { + "type": "string" + }, + "dedup_period": { + "type": "integer" + } + }, + "required": ["enabled", "dedup_key"] + }, + "router": { + "type": "object", + "properties": { + "topic": { + "type": "string", + "minLength": 1 + } + } + }, + "validation": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "mode": { + "type": "string" + } + }, + "required": ["enabled", "mode"] + } + }, + "required": ["batch", "dedup", "validation"] + }, + "transformations": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "key": { + "type": "string" + }, + "expr": { + "type": "string" + }, + "mode": { + "type": "string" + }, + "dataType": { + "type": "string" + }, + "section": { + "type": "string" + } + }, + "required": [ + "type", + "key", + "expr", + "mode", + "dataType", + "section" + ] + } + }, + "denorm": { + "type": "array", + "items": { + "type": "object", + "properties": { + "master_dataset_id": { + "type": "string" + }, + "denorm_key": { + "type": "string" + }, + "out": { + "type": "string" + } + }, + "required": ["master_dataset_id", "denorm_key", "out"] + } + } + }, + "required": ["metadata", "data_schema"], + "additionalProperties": false + } + }, + "required": ["data"] + } + }, + "required": ["id", "ver", "ts", "params", "request"], + "additionalProperties": false +} diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchema.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json similarity index 100% rename from api-service/src/controllers/DatasetImport/RequestValidationSchema.json rename to api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json From 232a4dbd65e50d72ec447c1d04d8a2791647ff3c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 30 Jul 2024 13:20:17 +0530 Subject: [PATCH 076/235] #OBS-I116: fix: Dataset overwrite after creation failure --- .../controllers/DatasetImport/DatasetImport.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 565550ee..bbef45f5 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -18,18 +18,22 @@ const datasetImport = async (req: Request, res: Response) => { const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) - const dataset = overwrite == "true" ? await overWriteDataset(updatedDataset) : await createNewDataset(updatedDataset) + const dataset = await importDataset(updatedDataset, overwrite); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); } -const createNewDataset = async (dataset: Record) => { - return datasetService.createDraftDataset(dataset).catch(err => { - if (err?.name === 'SequelizeUniqueConstraintError') { - const dataset_id = _.get(dataset, "dataset_id") - throw obsrvError(dataset.dataset_id, "DATASET_ALREADY_EXISTS", `Dataset with id : ${dataset_id} already exists to import`, "BAD_REQUEST", 400); +const importDataset = async (dataset: Record, overwrite: string | any) => { + const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) + if (response?.name === 'SequelizeUniqueConstraintError') { + if (overwrite == "true") { + const overwriteRes = await datasetService.updateDraftDataset(dataset) + return _.omit(overwriteRes, ["message"]) } + } + if(response?.errors){ throw obsrvError("", "DATASET_IMPORT_FAILURE", `Failed to import dataset`, "INTERNAL_SERVER_ERROR", 500); - }) + } + return response } const overWriteDataset = async (dataset: Record) => { From d400b0ed4a723d4db6a7ccd0532e5c9e13c11ad2 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 30 Jul 2024 13:22:29 +0530 Subject: [PATCH 077/235] #OBS-I116: fix: error handling --- api-service/src/controllers/DatasetImport/DatasetImport.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index bbef45f5..56aacf3a 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -26,7 +26,9 @@ const importDataset = async (dataset: Record, overwrite: string | a const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) if (response?.name === 'SequelizeUniqueConstraintError') { if (overwrite == "true") { - const overwriteRes = await datasetService.updateDraftDataset(dataset) + const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(err=>{ + throw obsrvError("", "DATASET_IMPORT_FAILURE", `Failed to import dataset`, "INTERNAL_SERVER_ERROR", 500); + }) return _.omit(overwriteRes, ["message"]) } } From 8a5646ad44d279d2d0ff4a3fe6c098b5f3e88341 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 30 Jul 2024 13:24:45 +0530 Subject: [PATCH 078/235] #OBS-I116: fix: error messages fix --- api-service/src/controllers/DatasetImport/DatasetImport.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 56aacf3a..3bb6165f 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -23,17 +23,18 @@ const datasetImport = async (req: Request, res: Response) => { } const importDataset = async (dataset: Record, overwrite: string | any) => { + const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) if (response?.name === 'SequelizeUniqueConstraintError') { if (overwrite == "true") { const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(err=>{ - throw obsrvError("", "DATASET_IMPORT_FAILURE", `Failed to import dataset`, "INTERNAL_SERVER_ERROR", 500); + throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) return _.omit(overwriteRes, ["message"]) } } if(response?.errors){ - throw obsrvError("", "DATASET_IMPORT_FAILURE", `Failed to import dataset`, "INTERNAL_SERVER_ERROR", 500); + throw obsrvError("", "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id}`, "INTERNAL_SERVER_ERROR", 500); } return response } From 6c4c9cb16b60d935cf5f26abdfa68bbc2e14141b Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 30 Jul 2024 13:30:54 +0530 Subject: [PATCH 079/235] #OBS-I116: fix: code fixes --- .../src/controllers/DatasetImport/DatasetImport.ts | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 3bb6165f..3f5ef29f 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -26,7 +26,7 @@ const importDataset = async (dataset: Record, overwrite: string | a const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) if (response?.name === 'SequelizeUniqueConstraintError') { - if (overwrite == "true") { + if (overwrite === "true") { const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(err=>{ throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) @@ -39,16 +39,6 @@ const importDataset = async (dataset: Record, overwrite: string | a return response } -const overWriteDataset = async (dataset: Record) => { - const dataset_id = _.get(dataset, "dataset_id") - const draftDataset = await datasetService.getDraftDataset(dataset_id, ["id"]) - if (!draftDataset) { - throw obsrvError(dataset.dataset_id, "DATASET_NOT_FOUND", `Dataset with dataset_id: ${dataset_id} not found to overwrite`, "NOT_FOUND", 404) - } - const response = await datasetService.updateDraftDataset(dataset) - return _.omit(response, ["message"]) -} - const getResponseData = (ignoredConfigs: Record) => { const { ignoredConnectors, ignoredTransformations, ignoredDenorms } = ignoredConfigs; let successMsg = "Dataset is imported successfully"; From ec4a1b8aae7efcf79d2b4051a9465c556e9973ce Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 30 Jul 2024 14:30:13 +0530 Subject: [PATCH 080/235] #OBS-I142: formatted connector list api files --- .../ConnectorsList/ConnectorsList.ts | 22 ++++----- .../ConnectorsListValidationSchema.json | 48 +++++++++++++------ api-service/src/services/ConnectorService.ts | 6 +-- 3 files changed, 47 insertions(+), 29 deletions(-) diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts index 61bee8bf..96fb76d2 100644 --- a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts +++ b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts @@ -8,9 +8,9 @@ import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; import { connectorService } from "../../services/ConnectorService"; -const defaultFeilds = ["id","connector_id","name", "type", "category", "version", "description", "technology", "runtime", "licence", "owner", "iconurl", "status", "created_by", "updated_by", "created_date", "updated_date"]; +const defaultFeilds = ["id", "connector_id", "name", "type", "category", "version", "description", "technology", "runtime", "licence", "owner", "iconurl", "status", "created_by", "updated_by", "created_date", "updated_date"]; -const validateRequest = (req: Request)=>{ +const validateRequest = (req: Request) => { const isRequestValid: Record = schemaValidation(req.body, ConnectorListSchema) if (!isRequestValid.isValid) { throw obsrvError("", "CONNECTORS_LIST_INPUT_INVALID", isRequestValid.message, "BAD_REQUEST", 400) @@ -19,28 +19,28 @@ const validateRequest = (req: Request)=>{ const connectorsList = async (req: Request, res: Response) => { validateRequest(req); - + const connectorBody = req.body.request; const connectorList = await listConnectors(connectorBody) const responseData = { data: connectorList, count: _.size(connectorList) } - logger.info({req: req.body, resmsgid: _.get(res, "resmsgid"), message: `Connectors are listed successfully with a connectors count (${_.size(connectorList)})` }) + logger.info({ req: req.body, resmsgid: _.get(res, "resmsgid"), message: `Connectors are listed successfully with a connectors count (${_.size(connectorList)})` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); } const listConnectors = async (request: Record): Promise> => { const { filters = {} } = request || {}; const connectorStatus = _.get(filters, "status"); - const connectorCategory = _.get(filters,"category"); - const filterOptions: any = {}; - if(!_.isEmpty(connectorStatus)){ + const connectorCategory = _.get(filters, "category"); + const filterOptions: any = {}; + if (!_.isEmpty(connectorStatus)) { filterOptions["status"] = connectorStatus } - if(!_.isEmpty(connectorCategory)){ - filterOptions["category"] =connectorCategory + if (!_.isEmpty(connectorCategory)) { + filterOptions["category"] = connectorCategory } - const filteredconnectorList = await connectorService.findConnectors(filterOptions,defaultFeilds); - return filteredconnectorList ; + const filteredconnectorList = await connectorService.findConnectors(filterOptions, defaultFeilds); + return filteredconnectorList; } export default connectorsList; \ No newline at end of file diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json b/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json index 65afbdeb..61847b06 100644 --- a/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json +++ b/api-service/src/controllers/ConnectorsList/ConnectorsListValidationSchema.json @@ -3,7 +3,9 @@ "properties": { "id": { "type": "string", - "enum": ["api.connectors.list"] + "enum": [ + "api.connectors.list" + ] }, "ver": { "type": "string" @@ -18,7 +20,9 @@ "type": "string" } }, - "required": ["msgid"], + "required": [ + "msgid" + ], "additionalProperties": false }, "request": { @@ -29,27 +33,41 @@ "properties": { "status": { "type": "array", - "items": { - "type": "string", - "enum": ["Draft", "InValidation", "Live", "Retired"] - }, - "minItems": 1 + "items": { + "type": "string", + "enum": [ + "Draft", + "InValidation", + "Live", + "Retired" + ] }, + "minItems": 1 + }, "category": { "type": "array", - "items": { - "type": "string", - "enum": ["Database", "File"] - }, - "minItems": 1 - } - }, + "items": { + "type": "string", + "enum": [ + "Database", + "File" + ] + }, + "minItems": 1 + } + }, "additionalProperties": false } }, "additionalProperties": false } }, - "required": ["id", "ver", "ts", "params", "request"], + "required": [ + "id", + "ver", + "ts", + "params", + "request" + ], "additionalProperties": false } \ No newline at end of file diff --git a/api-service/src/services/ConnectorService.ts b/api-service/src/services/ConnectorService.ts index e6fb2034..3b7a5340 100644 --- a/api-service/src/services/ConnectorService.ts +++ b/api-service/src/services/ConnectorService.ts @@ -1,11 +1,11 @@ import { ConnectorRegistry } from "../models/ConnectorRegistry"; -class ConnectorService{ +class ConnectorService { findConnectors = async (where?: Record, attributes?: string[]): Promise => { - return ConnectorRegistry.findAll({where, attributes, raw: true}); + return ConnectorRegistry.findAll({ where, attributes, raw: true }); } - + } export const connectorService = new ConnectorService(); \ No newline at end of file From e8d40ae3ee7cb760200493e4635ec49adaad182a Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 30 Jul 2024 14:36:03 +0530 Subject: [PATCH 081/235] #OBS-I142: formatted connector list api files --- api-service/src/controllers/ConnectorsList/ConnectorsList.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts index 96fb76d2..3e9cae44 100644 --- a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts +++ b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts @@ -8,7 +8,7 @@ import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; import { connectorService } from "../../services/ConnectorService"; -const defaultFeilds = ["id", "connector_id", "name", "type", "category", "version", "description", "technology", "runtime", "licence", "owner", "iconurl", "status", "created_by", "updated_by", "created_date", "updated_date"]; +const defaultFields = ["id", "connector_id", "name", "type", "category", "version", "description", "technology", "runtime", "licence", "owner", "iconurl", "status", "created_by", "updated_by", "created_date", "updated_date"]; const validateRequest = (req: Request) => { const isRequestValid: Record = schemaValidation(req.body, ConnectorListSchema) @@ -39,7 +39,7 @@ const listConnectors = async (request: Record): Promise Date: Tue, 30 Jul 2024 14:44:43 +0530 Subject: [PATCH 082/235] #OBS-I142: formatted connector list file --- .../ConnectorsList/ConnectorsList.ts | 22 ++++++------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts index 3e9cae44..a031d370 100644 --- a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts +++ b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts @@ -3,7 +3,6 @@ import ConnectorListSchema from "./ConnectorsListValidationSchema.json"; import { obsrvError } from "../../types/ObsrvError"; import { schemaValidation } from "../../services/ValidationService"; import _ from "lodash"; -import logger from "../../logger"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; import { connectorService } from "../../services/ConnectorService"; @@ -19,28 +18,21 @@ const validateRequest = (req: Request) => { const connectorsList = async (req: Request, res: Response) => { validateRequest(req); - - const connectorBody = req.body.request; + const connectorBody = _.get(req, ["body", "request"]); const connectorList = await listConnectors(connectorBody) const responseData = { data: connectorList, count: _.size(connectorList) } - logger.info({ req: req.body, resmsgid: _.get(res, "resmsgid"), message: `Connectors are listed successfully with a connectors count (${_.size(connectorList)})` }) ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: responseData }); } const listConnectors = async (request: Record): Promise> => { const { filters = {} } = request || {}; - const connectorStatus = _.get(filters, "status"); - const connectorCategory = _.get(filters, "category"); + const status = _.get(filters, "status"); + const category = _.get(filters, "category"); const filterOptions: any = {}; - if (!_.isEmpty(connectorStatus)) { - filterOptions["status"] = connectorStatus - } - - if (!_.isEmpty(connectorCategory)) { - filterOptions["category"] = connectorCategory - } - const filteredconnectorList = await connectorService.findConnectors(filterOptions, defaultFields); - return filteredconnectorList; + if (!_.isEmpty(status)) filterOptions["status"] = status; + if (!_.isEmpty(category)) filterOptions["category"] = category; + return connectorService.findConnectors(filterOptions, defaultFields); + } export default connectorsList; \ No newline at end of file From 935647c114f964f94cadae672c801719811aadca Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 30 Jul 2024 16:10:45 +0530 Subject: [PATCH 083/235] #OBS-I138: required changes for dataset for master dataset migration from v1 to v2 --- .../src/configs/DatasetConfigDefault.ts | 1 + .../DatasetStatusTransition.ts | 17 ++++++--- api-service/src/services/DatasetService.ts | 37 +++++++++++++++---- 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/api-service/src/configs/DatasetConfigDefault.ts b/api-service/src/configs/DatasetConfigDefault.ts index 376d9f54..27bdeda0 100644 --- a/api-service/src/configs/DatasetConfigDefault.ts +++ b/api-service/src/configs/DatasetConfigDefault.ts @@ -51,6 +51,7 @@ export const defaultDatasetConfig = { "entry_topic": config.telemetry_service_config.kafka.topics.createDataset, "status": DatasetStatus.Draft, "api_version": "v2", + "sample_data":{}, "version": 1, "created_by": "SYSTEM", "updated_by": "SYSTEM" diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 8a091e6e..47ea8bbc 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -54,7 +54,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); validateRequest(req, dataset_id); - const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) validateDataset(dataset, dataset_id, status); switch(status) { @@ -95,6 +95,9 @@ const readyForPublish = async (dataset: Record) => { let draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) + if (draftDataset?.type === 'master') { + defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); + } _.merge(draftDataset, defaultConfigs) const datasetValid: Record = schemaValidation(draftDataset, ReadyToPublishSchema) if (!datasetValid.isValid) { @@ -141,7 +144,7 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) statusCode: 409 } } - const masterDatasets = await datasetService.findDatasets({id: datasetIds, type: "master"}, ["id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ id: datasetIds, type: "master" }, ["id", "status", "dataset_config", "api_version"]) const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) let datasetStatus : Record = { @@ -180,8 +183,12 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) } const updateMasterDataConfig = async (draftDataset: Record) => { - if(draftDataset.type === 'master') { - if(draftDataset.dataset_config.cache_config.redis_db === 0) { + if (draftDataset.type === 'master') { + // _.set(draftDataset,"datasetConfig.cache_config",datasetDefaultConfig) + let dataset_config = _.get(draftDataset, "dataset_config") + const datasetCacheConfig = _.get(defaultDatasetConfig, "dataset_config.cache_config") + draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } + if (draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() if (_.isEmpty(results)) { throw { @@ -209,7 +216,7 @@ const canRetireIfMasterDataset = async (dataset: Record) => { if (dataset.type === DatasetType.master) { - const liveDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live }, ["denorm_config", "id", "status"]) || [] + const liveDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live }, ["dataset_config", "dataset_id"]) || [] const draftDatasets = await datasetService.findDraftDatasets({ status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, ["denorm_config", "id", "status"]) || [] const allDatasets = _.union(liveDatasets, draftDatasets) const extractDenormFields = _.map(allDatasets, function(depDataset) { diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 84b6e2cd..3939b64d 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -130,6 +130,7 @@ class DatasetService { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); await DatasetSourceConfigDraft.destroy({ where: { dataset_id }, transaction }); + await DatasourceDraft.destroy({ where: { dataset_id }, transaction }); await transaction.commit(); } catch (err) { await transaction.rollback(); @@ -187,6 +188,25 @@ class DatasetService { const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode"]); draftDataset["transformations_config"] = transformations } + const denormConfig = _.get(draftDataset, "denorm_config") + if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id","dataset_id", "status", "dataset_config", "api_version"]) + if (_.isEmpty(masterDatasets)) { + throw { code: "DEPENDENT_MASTER_DATA_NOT_FOUND", message: `The dependent dataset not found`, errCode: "NOT_FOUND", statusCode: 404 } + } + const updatedDenormFields = _.map(denormConfig.denorm_fields, field => { + const { redis_db, denorm_out_field, denorm_key } = field + let masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.cache_config.redis_db") === redis_db) + if(!masterConfig){ + masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.redis_db") === redis_db) + } + if (_.isEmpty(masterConfig)) { + throw { code: "DEPENDENT_MASTER_DATA_NOT_LIVE", message: `The dependent master dataset is not published`, errCode: "PRECONDITION_REQUIRED", statusCode: 428 } + } + return { denorm_key, denorm_out_field, dataset_id: _.get(masterConfig, "dataset_id") } + }) + draftDataset["denorm_config"] = { ...denormConfig, denorm_fields: updatedDenormFields } + } draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft @@ -203,12 +223,12 @@ class DatasetService { const { id } = dataset const transaction = await sequelize.transaction() try { - await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetDraft.destroy({ where: { id } , transaction}) + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasourceDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetDraft.destroy({ where: { id }, transaction }) await transaction.commit() - } catch (err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) } @@ -260,8 +280,9 @@ class DatasetService { await this.createDruidDataSource(draftDataset, transaction); } if(indexingConfig.lakehouse_enabled) { - const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id"], true); - if(liveDataset) { + const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id", "api_version"], true); + + if(liveDataset && liveDataset.api_version === "v2") { await this.updateHudiDataSource(draftDataset, transaction) } else { await this.createHudiDataSource(draftDataset, transaction) @@ -300,7 +321,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record - const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource.ingestion_spec, allFields, draftDatasource.datasource_ref); + const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } From 4edcdbd9b8181a02841525099e3fab0b8fb81380 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 30 Jul 2024 16:23:32 +0530 Subject: [PATCH 084/235] #OBS-I138: removed cors package --- api-service/package.json | 1 - api-service/src/app.ts | 3 --- 2 files changed, 4 deletions(-) diff --git a/api-service/package.json b/api-service/package.json index ff5a5831..6585711b 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -30,7 +30,6 @@ "axios": "^1.6.0", "body-parser": "^1.20.2", "compression": "^1.7.4", - "cors": "^2.8.5", "dateformat": "2.0.0", "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", diff --git a/api-service/src/app.ts b/api-service/src/app.ts index dd184e2b..ec2a10cd 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -8,11 +8,8 @@ import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; import { alertsRouter } from "./routes/AlertsRouter"; -// import cors from "cors"; -const cors = require("cors"); const app: Application = express(); -app.use(cors()); app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); From 9c9ab916429cf5c61e32b50db5d64e9f4d1d6899 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 30 Jul 2024 16:35:57 +0530 Subject: [PATCH 085/235] #OBS-I138: updated package json file --- api-service/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/package.json b/api-service/package.json index 6585711b..f1b143ac 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -5,7 +5,7 @@ "main": "dist/app.js", "scripts": { "start": "ts-node ./src/app.ts", - "test": "source .env.test && nyc mocha ./src/v2/tests/**/GenerateSignedURL.spec.ts --exit", + "test": "source .env.test && nyc mocha ./src/v1/test/*.spec.ts --exit && nyc mocha ./src/v2/tests/**/*.spec.ts --exit", "actions:test": "nyc mocha ./src/v1/test/*.spec.ts --exit", "actions:test:v2": "nyc mocha ./src/v2/tests/**/*.spec.ts --exit", "build": "rm -rf dist && tsc --declaration -P . && cp package.json ./dist/package.json", @@ -19,10 +19,11 @@ "@aws-sdk/abort-controller": "^3.374.0", "@aws-sdk/client-s3": "^3.540.0", "@aws-sdk/credential-providers": "^3.309.0", - "@aws-sdk/lib-storage": "^3.609.0", + "@aws-sdk/lib-storage": "^3.182.0", "@aws-sdk/s3-request-presigner": "^3.540.0", "@azure/storage-blob": "^12.17.0", "@google-cloud/storage": "^7.9.0", + "@jsonhero/schema-infer": "^0.1.5", "@project-sunbird/logger": "^0.0.9", "ajv": "^8.11.2", "ajv-formats": "^2.1.1", @@ -44,7 +45,6 @@ "moment": "^2.29.4", "multiparty": "4.2.1", "node-sql-parser": "^5.1.0", - "obsrv-config-service-ext": "file:obsrv-config-service-ext-1.0.0.tgz", "pg": "^8.11.3", "pg-hstore": "^2.3.4", "prom-client": "^14.2.0", From e5bc628521cf4719811fba41a7d9058cad9c6c2e Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 30 Jul 2024 16:37:11 +0530 Subject: [PATCH 086/235] #OBS-I138: indentation fix --- api-service/package.json | 1 - api-service/src/app.ts | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/package.json b/api-service/package.json index f1b143ac..f760df95 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -16,7 +16,6 @@ "author": "Obsrv", "license": "MIT", "dependencies": { - "@aws-sdk/abort-controller": "^3.374.0", "@aws-sdk/client-s3": "^3.540.0", "@aws-sdk/credential-providers": "^3.309.0", "@aws-sdk/lib-storage": "^3.182.0", diff --git a/api-service/src/app.ts b/api-service/src/app.ts index ec2a10cd..f39784f2 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -10,6 +10,7 @@ import { config } from "./configs/Config"; import { alertsRouter } from "./routes/AlertsRouter"; const app: Application = express(); + app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); From a8c4c27f42cccf900daf147ea0ff845d72b8e550 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 30 Jul 2024 16:37:40 +0530 Subject: [PATCH 087/235] #OBS-I138: indentation fix --- api-service/src/app.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index f39784f2..ec2a10cd 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -10,7 +10,6 @@ import { config } from "./configs/Config"; import { alertsRouter } from "./routes/AlertsRouter"; const app: Application = express(); - app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); From 7f3fa1329e01a8687b2c286c8ed6c3452cb1600c Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 30 Jul 2024 16:41:19 +0530 Subject: [PATCH 088/235] #OBS-I138: fixed indentations --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 8 ++++---- api-service/src/services/DatasetService.ts | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 47ea8bbc..65a175fe 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -24,7 +24,7 @@ const allowedTransitions: Record = { } const liveDatasetActions = ["Retire", "Archive", "Purge"] -const validateRequest = (req: Request, datasetId: any) => { +const validateRequest = (req: Request, datasetId: any) => { const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) if (!isRequestValid.isValid) { throw obsrvError(datasetId, invalidRequest, isRequestValid.message, "BAD_REQUEST", 400) @@ -54,7 +54,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); validateRequest(req, dataset_id); - const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) validateDataset(dataset, dataset_id, status); switch(status) { @@ -144,7 +144,7 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) statusCode: 409 } } - const masterDatasets = await datasetService.findDatasets({ id: datasetIds, type: "master" }, ["id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({id: datasetIds, type: "master"}, ["id", "status", "dataset_config", "api_version"]) const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) let datasetStatus : Record = { @@ -190,7 +190,7 @@ const updateMasterDataConfig = async (draftDataset: Record) => { draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } if (draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() - if (_.isEmpty(results)) { + if(_.isEmpty(results)) { throw { code: "REDIS_DB_INDEX_FETCH_FAILED", message: `Unable to fetch the redis db index for the master data`, diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 3939b64d..eca4bb67 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -223,12 +223,12 @@ class DatasetService { const { id } = dataset const transaction = await sequelize.transaction() try { - await DatasetTransformationsDraft.destroy({ where: { dataset_id: id }, transaction }) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id }, transaction }) - await DatasourceDraft.destroy({ where: { dataset_id: id }, transaction }) - await DatasetDraft.destroy({ where: { id }, transaction }) + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) + await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) + await DatasetDraft.destroy({ where: { id } , transaction}) await transaction.commit() - } catch (err: any) { + } catch (err:any) { await transaction.rollback() throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) } From fee08a78dd9b058a85b26664ccbdbf2c6892261b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 30 Jul 2024 17:10:59 +0530 Subject: [PATCH 089/235] #OBS-I142: updated postman collection --- .../V2 docs.postman_collection.json | 2543 +++++++++++++++++ 1 file changed, 2543 insertions(+) create mode 100644 api-service/postman-collection/V2 docs.postman_collection.json diff --git a/api-service/postman-collection/V2 docs.postman_collection.json b/api-service/postman-collection/V2 docs.postman_collection.json new file mode 100644 index 00000000..b673ccfb --- /dev/null +++ b/api-service/postman-collection/V2 docs.postman_collection.json @@ -0,0 +1,2543 @@ +{ + "info": { + "_postman_id": "51471244-e9b6-453e-964d-dccbdac80e72", + "name": "V2 docs", + "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", + "_exporter_id": "37164806" + }, + "item": [ + { + "name": "Dataset api's", + "item": [ + { + "name": "Dataset create", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "response": [ + { + "name": "Success: Dataset created successfullly", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "287" + }, + { + "key": "ETag", + "value": "W/\"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ\"" + }, + { + "key": "Date", + "value": "Mon, 15 Jul 2024 13:14:09 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-15T18:44:08+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"276c042c-0f23-4b26-9b10-6fe48bbc2d3a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\"\n }\n}" + }, + { + "name": "Success: Master dataset created successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "291" + }, + { + "key": "ETag", + "value": "W/\"123-ZAXtVh5dFh84bZdu3SFBTQDhEHI\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:06:40 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:36:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"845076be-d9e7-4246-bb8e-07ae0ce59d1e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-master\",\n \"version_key\": \"1721099200603\"\n }\n}" + }, + { + "name": "Failure: Master dataset already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "337" + }, + { + "key": "ETag", + "value": "W/\"151-a7dJ9XBUyT3AXNxl1TPcraxMX08\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:07:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:37:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"138b796b-1b68-481a-a59d-1cb695c1adc9\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-master\"\n }\n}" + }, + { + "name": "Failure: Dataset already exists", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "333" + }, + { + "key": "ETag", + "value": "W/\"14d-QLXc3MJG4hFiMbBoA6mXFDpSryQ\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:08:05 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:38:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf62693c-3aa4-42ce-a5ea-4bde340740f5\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-t4\"\n }\n}" + }, + { + "name": "Failure: Invalid request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "362" + }, + { + "key": "ETag", + "value": "W/\"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:09:00 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:39:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a07de860-dcbc-4ff6-822e-34b47635c8a3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + }, + { + "name": "Success: Minimal request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_events\",\n \"type\":\"event\", //\"master\" for master dataset\n \"name\": \"sb-telemetry\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "284" + }, + { + "key": "ETag", + "value": "W/\"11c-kOoTX1K7Zbp+vsGfNEv87FBJOWg\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:44:59 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:14:59+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e207f4f-2be6-4a45-ab78-213bea272ae0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_events\",\n \"version_key\": \"1721133899306\"\n }\n}" + }, + { + "name": "Success: Dataset created successfully with all fields", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"/path/to/file1.csv\",\n \"/path/to/file2.csv\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": false,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"data_key\",\n \"partition_key\": \"partition_key\",\n \"timestamp_key\": \"time\",\n \"timestamp_format\": \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\": [\n \"tag1\"\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/create" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "288" + }, + { + "key": "ETag", + "value": "W/\"120-QhzA7Fga/ADDeU90Nohadt+J8fg\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:49:53 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:19:53+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"505fb3bc-ae32-4f5b-a931-adec4d1d84ba\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t41\",\n \"version_key\": \"1721220593027\"\n }\n}" + } + ] + }, + { + "name": "File generate url", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "response": [ + { + "name": "Success: Generate put url", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1344" + }, + { + "key": "ETag", + "value": "W/\"540-790rZel+H/rDwgvZRxvlUmZ8Gpc\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 02:56:19 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:26:19+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5306f309-4a15-458e-89e2-29d8ac0835d4\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry_10d595.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data_33109a.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject\"\n }\n ]\n}" + }, + { + "name": "Success: Generate get url", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1316" + }, + { + "key": "ETag", + "value": "W/\"524-iPflsPqFFV7NFaQCi2ODhQtbq/g\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 04:01:40 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"009c0b2d-8acd-40b0-a807-bbacf9242771\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject\"\n }\n ]\n}" + }, + { + "name": "Failure: limit exceeds", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "355" + }, + { + "key": "ETag", + "value": "W/\"163-9oQYJJEaBH3mJAnzDHXn2MxE848\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:03:04 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:33:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d3a606ca-47d0-4746-95a1-c8692e749959\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_URL_GENERATION_LIMIT_EXCEED\",\n \"message\": \"Pre-signed URL generation failed: limit exceeded.\",\n \"trace\": \"\"\n }\n}" + }, + { + "name": "Failure: Invalid request", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"update\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/files/generate-url" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "390" + }, + { + "key": "ETag", + "value": "W/\"186-KH9x0zC3+RqtWpa0tVT9XVg8agk\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 04:01:10 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"c3e9da1c-09f3-4a3b-84ec-a19efc68b856\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_GENERATE_URL_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/access/enum must be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" + } + ] + }, + { + "name": "Data schema generator", + "request": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json, text/plain, */*", + "disabled": true + }, + { + "key": "Accept-Language", + "value": "en-GB,en", + "disabled": true + }, + { + "key": "Cache-Control", + "value": "no-store", + "disabled": true + }, + { + "key": "Connection", + "value": "keep-alive", + "disabled": true + }, + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", + "disabled": true + }, + { + "key": "Origin", + "value": "http://localhost:3001", + "disabled": true + }, + { + "key": "Pragma", + "value": "no-store", + "disabled": true + }, + { + "key": "Referer", + "value": "http://localhost:3001/console/dataset/new", + "disabled": true + }, + { + "key": "Sec-Fetch-Dest", + "value": "empty", + "disabled": true + }, + { + "key": "Sec-Fetch-Mode", + "value": "cors", + "disabled": true + }, + { + "key": "Sec-Fetch-Site", + "value": "same-origin", + "disabled": true + }, + { + "key": "Sec-GPC", + "value": "1", + "disabled": true + }, + { + "key": "User-Agent", + "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "disabled": true + }, + { + "key": "sec-ch-ua", + "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", + "disabled": true + }, + { + "key": "sec-ch-ua-mobile", + "value": "?0", + "disabled": true + }, + { + "key": "sec-ch-ua-platform", + "value": "\"macOS\"", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3007/dataset/v1/dataschema" + }, + "response": [ + { + "name": "Success: Schema generated successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Accept", + "value": "application/json, text/plain, */*", + "disabled": true + }, + { + "key": "Accept-Language", + "value": "en-GB,en", + "disabled": true + }, + { + "key": "Cache-Control", + "value": "no-store", + "disabled": true + }, + { + "key": "Connection", + "value": "keep-alive", + "disabled": true + }, + { + "key": "Content-Type", + "value": "application/json", + "disabled": true + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", + "disabled": true + }, + { + "key": "Origin", + "value": "http://localhost:3001", + "disabled": true + }, + { + "key": "Pragma", + "value": "no-store", + "disabled": true + }, + { + "key": "Referer", + "value": "http://localhost:3001/console/dataset/new", + "disabled": true + }, + { + "key": "Sec-Fetch-Dest", + "value": "empty", + "disabled": true + }, + { + "key": "Sec-Fetch-Mode", + "value": "cors", + "disabled": true + }, + { + "key": "Sec-Fetch-Site", + "value": "same-origin", + "disabled": true + }, + { + "key": "Sec-GPC", + "value": "1", + "disabled": true + }, + { + "key": "User-Agent", + "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", + "disabled": true + }, + { + "key": "sec-ch-ua", + "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", + "disabled": true + }, + { + "key": "sec-ch-ua-mobile", + "value": "?0", + "disabled": true + }, + { + "key": "sec-ch-ua-platform", + "value": "\"macOS\"", + "disabled": true + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3007/dataset/v1/dataschema" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4414" + }, + { + "key": "ETag", + "value": "W/\"113e-ykaeY2EqBHdGqGLcr7K3WSs5fYo\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 03:28:22 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"dataset.schema.identify\",\n \"ver\": \"v1\",\n \"ts\": 1721100502574,\n \"params\": {\n \"status\": \"SUCCESS\",\n \"errmsg\": \"\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tripID' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tripID\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"VendorID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tpep_pickup_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_pickup_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_pickup_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"tpep_dropoff_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_dropoff_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_dropoff_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"passenger_count\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"trip_distance\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"RatecodeID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"store_and_fwd_flag\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"PULocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"DOLocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"payment_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primary_passenger\": {\n \"type\": \"object\",\n \"properties\": {\n \"email\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mobile\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"fare_details\": {\n \"type\": \"object\",\n \"properties\": {\n \"fare_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"extra\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mta_tax\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tip_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tolls_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"improvement_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"total_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"congestion_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 98,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + } + ] + }, + { + "name": "Dataset update", + "request": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721135455988\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "response": [ + { + "name": "Success: Minimal dataset update", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "331" + }, + { + "key": "ETag", + "value": "W/\"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 13:00:45 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:30:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"354f1fec-0c39-42ee-a52a-49552f847c11\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134845559\"\n }\n}" + }, + { + "name": "Success: Updated successfully", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "331" + }, + { + "key": "ETag", + "value": "W/\"14b-y8oEEJijvIDh8wU5ogipKdkv8y0\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:57:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:27:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"6d835f07-aed5-4e8b-81c2-2142cfb55c52\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\"\n }\n}" + }, + { + "name": "Failure: Outdated key provided", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721064642580\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Conflict", + "code": 409, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "383" + }, + { + "key": "ETag", + "value": "W/\"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:53:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:23:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fe03f6-c4c4-48f6-9d84-a32cd52f4c13\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_OUTDATED\",\n \"message\": \"The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates\"\n }\n}" + }, + { + "name": "Failure: Dataset not exists to update", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "340" + }, + { + "key": "ETag", + "value": "W/\"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:58:30 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:28:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf64703c-bb6b-41bf-bc1a-c85373efd925\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists with id:telemetry_record-t41\"\n }\n}" + }, + { + "name": "Failure: Invalid request", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "369" + }, + { + "key": "ETag", + "value": "W/\"171-iNJoyWUecOEsXbHZwx6rld3Sr1I\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 12:59:21 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:29:21+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d31672b-e5c3-4a6d-afac-d9d78011bcde\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" + }, + { + "name": "Failure: No fields are provided to update", + "originalRequest": { + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/update" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "373" + }, + { + "key": "ETag", + "value": "W/\"175-ga30XLi7qSW4Ix+3Q/xaMUYcqII\"" + }, + { + "key": "Date", + "value": "Tue, 16 Jul 2024 13:02:44 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:32:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf99b1e1-7694-4be0-ba5d-e347764736de\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\"\n }\n}" + } + ] + }, + { + "name": "Read dataset", + "request": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "beckn-test-data" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "response": [ + { + "name": "Success: Read live dataset", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": "localhost:3007/v2/datasets/read/master-test" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "501" + }, + { + "key": "ETag", + "value": "W/\"1f5-p+b/6r0nHRFhgr5+URzxk4d/CSg\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:08:55 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"8c8a2852-54bc-43fb-b063-7f359d11930a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n }\n}" + }, + { + "name": "Success: Read draft dataset", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "beckn-test-data" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "528" + }, + { + "key": "ETag", + "value": "W/\"210-Qo8q3dU8l7LYIXVzJwStVe90z9Q\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:11:00 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:41:00+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"96fd4f42-fa84-4730-bc79-d241a4e335a1\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n }\n}" + }, + { + "name": "Success: Read specific column", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/master-test?status=Draft&fields=name,type,id", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "master-test" + ], + "query": [ + { + "key": "status", + "value": "Draft" + }, + { + "key": "fields", + "value": "name,type,id" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "237" + }, + { + "key": "ETag", + "value": "W/\"ed-zvknH4AY6kid9Yit+KqMIdDeNGc\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:12:16 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:42:16+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"02a6b03a-8bf3-4e37-8dcd-859d3e8f904e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"id\": \"master-test\"\n }\n}" + }, + { + "name": "Success: Read version_key", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/beckn-test-data?fields=version_key&mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "beckn-test-data" + ], + "query": [ + { + "key": "fields", + "value": "version_key" + }, + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "211" + }, + { + "key": "ETag", + "value": "W/\"d3-hfyvp0pFPZhM2NbiiMDTfy+51YM\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:15:37 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:45:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"805e848a-d260-47c3-b55c-fc9b8323719e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1718791650227\"\n }\n}" + }, + { + "name": "Success: Read from draft, if not present cerate draft from live dataset and then read", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/sample1?mode=edit", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "sample1" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "538" + }, + { + "key": "ETag", + "value": "W/\"21a-Iqg1rOMRqVpT0n8ZhQsPiTB2l44\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:19:28 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:49:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da70e25b-6ad0-49a7-a39d-340d1d0c46a7\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 2,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n }\n }\n}" + }, + { + "name": "Failure: Invalid field name provided", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": { + "raw": "localhost:3007/v2/datasets/read/telemetry_record?fields=newname", + "host": [ + "localhost" + ], + "port": "3007", + "path": [ + "v2", + "datasets", + "read", + "telemetry_record" + ], + "query": [ + { + "key": "fields", + "value": "newname" + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "307" + }, + { + "key": "ETag", + "value": "W/\"133-TQ9WpmutsrDcTNkRRmbWOhUChMk\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:20:17 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:50:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\"\n }\n}" + }, + { + "name": "Failure: Dataset not found", + "originalRequest": { + "method": "GET", + "header": [ + { + "key": "Cookie", + "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" + } + ], + "url": "localhost:3007/v2/datasets/read/new_telemetry_record.1" + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "304" + }, + { + "key": "ETag", + "value": "W/\"130-JL/oBB+GUHTrBp278giBHRvO71I\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:21:12 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:51:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"3aad3842-a76e-4fe8-b635-c7fef5f318f9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:new_telemetry_record.1 not found\"\n }\n}" + } + ] + }, + { + "name": "Dataset list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "response": [ + { + "name": "Success: Lists all when no filters provided", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "24340" + }, + { + "key": "ETag", + "value": "W/\"5f14-Cq3tfdk3YuXhXtjub1V0q8YVdC4\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:25:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:55:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"97efe04d-e981-493d-9ee7-a6dad6887d64\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry_record-t4\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_events\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_record-master\",\n \"name\": \"sb-telemetry\",\n \"type\": \"master\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"generate-schema\",\n \"name\": \"generate-schema\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-test\",\n \"name\": \"trip-test\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample-trip-details\",\n \"name\": \"sample-trip-details\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 30\n }\n}" + }, + { + "name": "Success: Filter based on status as array", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "5475" + }, + { + "key": "ETag", + "value": "W/\"1563-FnaUnwhv4LmMcE5J8f4+jKz+DDk\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:27:38 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" + }, + { + "name": "Success: Filter basen on status as string", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"ReadyToPublish\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "16960" + }, + { + "key": "ETag", + "value": "W/\"4240-18LMqkfsWst/M+Uc53td59PATTk\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:29:18 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:59:18+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a08c7ea0-bb1c-4998-b47d-a76e38e87e31\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 8\n }\n}" + }, + { + "name": "Success: Filter based on dataset type", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Live\",\n \"type\": \"master\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "933" + }, + { + "key": "ETag", + "value": "W/\"3a5-IM9o2EpRUzccL+l2CW/BBpoBMqA\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:30:41 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:00:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0d1ff2de-42c9-4192-b75d-84f711dbfb55\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n }\n ],\n \"count\": 2\n }\n}" + }, + { + "name": "Failure: Invalid payload", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/list" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "312" + }, + { + "key": "ETag", + "value": "W/\"138-XQplwhrgIYKIg0qtQdRCYWIGTNM\"" + }, + { + "key": "Date", + "value": "Wed, 17 Jul 2024 12:32:26 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:02:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"add9dbe0-f362-4f99-890c-3387c998a049\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/required must have required property 'msgid'\"\n }\n}" + } + ] + }, + { + "name": "Data schema generator", + "request": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ],\n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "response": [ + { + "name": "Data schema generated successfully", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ],\n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "10760" + }, + { + "key": "ETag", + "value": "W/\"2a08-QF5x1q0kIlfE9XU/pa9IboJuY8I\"" + }, + { + "key": "Date", + "value": "Mon, 22 Jul 2024 07:02:50 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:32:50+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1309aea0-9a97-46e9-bc5e-a16a8a7fb624\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'sid'. The property sid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.sid\"\n },\n {\n \"message\": \"The Property 'context.sid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.sid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.cdata[*].id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.cdata.items.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'l1'. The property l1: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.rollup.properties.l1\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uid'. The property uid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.uid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"object\": {\n \"type\": \"object\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'object'. The property object: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.object\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'tags'. The property tags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.tags\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'pageid'. The property pageid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.pageid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'subtype'. The property subtype: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.subtype\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uri'. The property uri: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.uri\"\n },\n {\n \"message\": \"The Property 'edata.uri' appears to be 'uri' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.uri\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"visits\": {\n \"type\": \"array\",\n \"items\": false,\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'visits'. The property visits: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.visits\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"level\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"message\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"params\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'syncts'. The property syncts: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.syncts\"\n },\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: '@timestamp'. The property @timestamp: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.@timestamp\"\n },\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'ex_processed'. The property ex_processed: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags.properties.ex_processed\"\n }\n ],\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'flags'. The property flags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"mid\": {\n \"path\": \"$.mid\",\n \"cardinality\": 67,\n \"index\": false\n },\n \"actor.id\": {\n \"path\": \"$.actor.properties.id\",\n \"cardinality\": 56,\n \"index\": false\n },\n \"context.sid\": {\n \"path\": \"$.context.properties.sid\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"edata.uri\": {\n \"path\": \"$.edata.properties.uri\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"context.cdata[*].id\": {\n \"path\": \"$.context.properties.cdata.items.properties.id\",\n \"cardinality\": 62,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"mid\",\n \"context.cdata[*].id\",\n \"actor.id\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" + }, + { + "name": "Failure: Invalid request body", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "364" + }, + { + "key": "ETag", + "value": "W/\"16c-tfKVtCWTjNkWCtH8cFw1RrzbgV0\"" + }, + { + "key": "Date", + "value": "Mon, 22 Jul 2024 07:03:47 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:33:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bbcc86c2-042d-4f77-bb6e-e1c9116df570\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'data'\"\n }\n}" + }, + { + "name": "Failure: Invalid request (config not provided)", + "originalRequest": { + "method": "POST", + "header": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "Cookie", + "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" + } + ], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ]\n \n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "localhost:3007/v2/datasets/dataschema" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "366" + }, + { + "key": "ETag", + "value": "W/\"16e-YeX++2sGUWsjHqjP2GoTgqujU+c\"" + }, + { + "key": "Date", + "value": "Mon, 22 Jul 2024 07:05:36 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:35:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1f856c5e-37f0-41e9-96fb-642471228da2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'config'\"\n }\n}" + } + ] + } + ] + }, + { + "name": "Connector api's", + "item": [ + { + "name": "Connector list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "response": [ + { + "name": "Success: Filtered based on category", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6dc\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1908" + }, + { + "key": "ETag", + "value": "W/\"774-58gfxDsxY66h3KZnTg62QbkxcPY\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:37:42 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:07:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"abc\",\n \"resmsgid\": \"632d3342-fd8a-47f7-afbb-96402a00b92f\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 3\n }\n}" + }, + { + "name": "Failure: Invalid request body, filter option array should not be empty", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "399" + }, + { + "key": "ETag", + "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:43:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + }, + { + "name": "Success: Connectors list with all filter options", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4107" + }, + { + "key": "ETag", + "value": "W/\"100b-SEJt8xn1TwmZz8DKpFAwkYx6E8s\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:47:51 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:17:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"fffa3ee0-da12-4bea-9b72-365571a62a4e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + }, + { + "name": "Success: Connectors list without filters", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4107" + }, + { + "key": "ETag", + "value": "W/\"100b-N8mD4H5+SlIOmnNcq6Fnlq2pwXs\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:50:11 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:20:11+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c527a46-7bfe-4c74-a290-4dcf1a3c5f10\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + }, + { + "name": "Success: Filtered based on status", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "250" + }, + { + "key": "ETag", + "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:55:51 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + } + ] + } + ] + } + ] +} \ No newline at end of file From 1ae38803ddd5b75c974eaf6d651459ba176c4cfe Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 30 Jul 2024 17:11:44 +0530 Subject: [PATCH 090/235] #OBS-I142: added connector list swagger documentation --- .../swagger-doc/v2_updated_doc_openapi.yml | 483 +++++++++++++++++- 1 file changed, 482 insertions(+), 1 deletion(-) diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml index ea2a663c..48400bff 100644 --- a/api-service/swagger-doc/v2_updated_doc_openapi.yml +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -4140,4 +4140,485 @@ paths: message: >- #properties/request/properties/access/enum must be equal to one of the allowed values - trace: '' \ No newline at end of file + trace: '' + /v2/connectors/list: + post: + tags: + - Connector api's + summary: Connector list + requestBody: + content: + application/json: + examples: + example1: + summary: Filter based on category + value: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + category: + - Database + example2: + summary: Connectors list with all filter options + value: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + category: + - Database, File + status: + - Live, Draft, InValidation, Retired + example3: + summary: Connectors list without filters + value: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: {} + example4: + summary: Filtered based on status + value: + id: api.connectors.list + ver: v2 + ts: '2024-04-10T16:10:50+05:30' + params: + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + request: + filters: + status: + - Draft + responses: + '200': + description: OK + content: + application/json: + schema: + type: object + examples: + example-0: + summary: 'Success: Filtered based on category' + value: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:07:42+05:30' + params: + status: SUCCESS + msgid: abc + resmsgid: 632d3342-fd8a-47f7-afbb-96402a00b92f + responseCode: OK + result: + data: + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + count: 3 + example-1: + summary: 'Success: Connectors list with all filter options' + value: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:17:51+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: fffa3ee0-da12-4bea-9b72-365571a62a4e + responseCode: OK + result: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + count: 7 + example-2: + summary: 'Success: Connectors list without filters' + value: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:20:11+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 2c527a46-7bfe-4c74-a290-4dcf1a3c5f10 + responseCode: OK + result: + data: + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL + type: source + category: Database + version: 1.0.0 + description: >- + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle + type: source + category: Database + version: 1.0.0 + description: >- + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + - id: aws-s3-connector-0.1.0 + connector_id: aws-s3-connector + name: AWS S3 + type: source + category: File + version: 0.1.0 + description: >- + The AWS S3 Connector is used to move data from any + S3 Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.237Z' + updated_date: '2024-06-25T04:39:21.237Z' + - id: azure-blob-connector-0.1.0 + connector_id: azure-blob-connector + name: Azure Blob Store + type: source + category: File + version: 0.1.0 + description: >- + The Azure Blob Store Connector is used to move data + from any Azure Blob Container to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.302Z' + updated_date: '2024-06-25T04:39:21.302Z' + - id: gcs-connector-0.1.0 + connector_id: gcs-connector + name: Google Cloud Storage + type: source + category: File + version: 0.1.0 + description: >- + The GCS Connector is used to move data from any + Google Bucket to the Obsrv platform + technology: python + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:39:21.364Z' + updated_date: '2024-06-25T04:39:21.364Z' + count: 7 + example-3: + summary: 'Success: Filtered based on status' + value: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:25:51+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: f506e725-eed4-41df-86dc-2477d5c4d19a + responseCode: OK + result: + data: [] + count: 0 + '400': + description: Bad Request + content: + application/json: + schema: + type: object + example: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:13:14+05:30' + params: + status: FAILED + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: 02fadde0-8c59-4420-8ab3-56474b01670b + responseCode: BAD_REQUEST + result: {} + error: + code: CONNECTORS_LIST_INPUT_INVALID + message: >- + #properties/request/properties/filters/properties/status/minItems + must NOT have fewer than 1 items \ No newline at end of file From 8a4578e199a7e5be0c13ad834cab05a8aabb8645 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 30 Jul 2024 17:57:58 +0530 Subject: [PATCH 091/235] #OBS-I142: updated postman collection --- .../V2 docs.postman_collection.json | 2543 ----------------- .../updated_v2_collection.json | 289 +- 2 files changed, 286 insertions(+), 2546 deletions(-) delete mode 100644 api-service/postman-collection/V2 docs.postman_collection.json diff --git a/api-service/postman-collection/V2 docs.postman_collection.json b/api-service/postman-collection/V2 docs.postman_collection.json deleted file mode 100644 index b673ccfb..00000000 --- a/api-service/postman-collection/V2 docs.postman_collection.json +++ /dev/null @@ -1,2543 +0,0 @@ -{ - "info": { - "_postman_id": "51471244-e9b6-453e-964d-dccbdac80e72", - "name": "V2 docs", - "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "37164806" - }, - "item": [ - { - "name": "Dataset api's", - "item": [ - { - "name": "Dataset create", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/create" - }, - "response": [ - { - "name": "Success: Dataset created successfullly", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/create" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "287" - }, - { - "key": "ETag", - "value": "W/\"11f-uBTr0zBIIFpz/sdLJx6WQf0rAbQ\"" - }, - { - "key": "Date", - "value": "Mon, 15 Jul 2024 13:14:09 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-15T18:44:08+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"276c042c-0f23-4b26-9b10-6fe48bbc2d3a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\"\n }\n}" - }, - { - "name": "Success: Master dataset created successfully", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/create" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "291" - }, - { - "key": "ETag", - "value": "W/\"123-ZAXtVh5dFh84bZdu3SFBTQDhEHI\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 03:06:40 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:36:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"845076be-d9e7-4246-bb8e-07ae0ce59d1e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-master\",\n \"version_key\": \"1721099200603\"\n }\n}" - }, - { - "name": "Failure: Master dataset already exists", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-master\",\n \"type\": \"master\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/create" - }, - "status": "Conflict", - "code": 409, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "337" - }, - { - "key": "ETag", - "value": "W/\"151-a7dJ9XBUyT3AXNxl1TPcraxMX08\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 03:07:28 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:37:28+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"138b796b-1b68-481a-a59d-1cb695c1adc9\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-master\"\n }\n}" - }, - { - "name": "Failure: Dataset already exists", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/create" - }, - "status": "Conflict", - "code": 409, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "333" - }, - { - "key": "ETag", - "value": "W/\"14d-QLXc3MJG4hFiMbBoA6mXFDpSryQ\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 03:08:05 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:38:05+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf62693c-3aa4-42ce-a5ea-4bde340740f5\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_EXISTS\",\n \"message\": \"Dataset Already exists with id:telemetry_record-t4\"\n }\n}" - }, - { - "name": "Failure: Invalid request body", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t4\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\":[\"tag1\"]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/create" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "362" - }, - { - "key": "ETag", - "value": "W/\"16a-Jn1DYy5EYoYF/Syd3f9LOvOK0lI\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 03:09:00 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:39:00+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a07de860-dcbc-4ff6-822e-34b47635c8a3\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" - }, - { - "name": "Success: Minimal request body", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_events\",\n \"type\":\"event\", //\"master\" for master dataset\n \"name\": \"sb-telemetry\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/create" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "284" - }, - { - "key": "ETag", - "value": "W/\"11c-kOoTX1K7Zbp+vsGfNEv87FBJOWg\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 12:44:59 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:14:59+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"9e207f4f-2be6-4a45-ab78-213bea272ae0\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_events\",\n \"version_key\": \"1721133899306\"\n }\n}" - }, - { - "name": "Success: Dataset created successfully with all fields", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"type\": \"event\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"id\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"dataset_config\": {\n \"file_upload_path\": [\n \"/path/to/file1.csv\",\n \"/path/to/file2.csv\"\n ],\n \"indexing_config\": {\n \"olap_store_enabled\": false,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"data_key\",\n \"partition_key\": \"partition_key\",\n \"timestamp_key\": \"time\",\n \"timestamp_format\": \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"\n }\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\",\n \"dataset_id\": \"master-telemetry\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\": \"string\",\n \"category\": \"pii\"\n },\n \"mode\": \"Strict\"\n }\n ],\n \"tags\": [\n \"tag1\"\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/create" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "288" - }, - { - "key": "ETag", - "value": "W/\"120-QhzA7Fga/ADDeU90Nohadt+J8fg\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:49:53 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.create\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:19:53+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"505fb3bc-ae32-4f5b-a931-adec4d1d84ba\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"telemetry_record-t41\",\n \"version_key\": \"1721220593027\"\n }\n}" - } - ] - }, - { - "name": "File generate url", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/files/generate-url" - }, - "response": [ - { - "name": "Success: Generate put url", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"write\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/files/generate-url" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "1344" - }, - { - "key": "ETag", - "value": "W/\"540-790rZel+H/rDwgvZRxvlUmZ8Gpc\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 02:56:19 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:26:19+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"5306f309-4a15-458e-89e2-29d8ac0835d4\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry_10d595.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry_10d595.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=49bbe1fe3fb1a16a0baa07ecd7331d9f6500c476287d225077f1a5dbccddeb50&X-Amz-SignedHeaders=host&x-id=PutObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data_33109a.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data_33109a.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T025619Z&X-Amz-Expires=600&X-Amz-Signature=5ece002651b6437caa0193b5241a9172faec600093e4dca7f831645004c38cf5&X-Amz-SignedHeaders=host&x-id=PutObject\"\n }\n ]\n}" - }, - { - "name": "Success: Generate get url", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/files/generate-url" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "1316" - }, - { - "key": "ETag", - "value": "W/\"524-iPflsPqFFV7NFaQCi2ODhQtbq/g\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 04:01:40 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:40+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"009c0b2d-8acd-40b0-a807-bbacf9242771\"\n },\n \"responseCode\": \"OK\",\n \"result\": [\n {\n \"filePath\": \"test-connector/api-service/user_uploads/telemetry.json\",\n \"fileName\": \"telemetry.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/telemetry.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=f14978e897a7a15f23afb1ef9496d187a2f21abfb71c55a568461be4c5688cc6&X-Amz-SignedHeaders=host&x-id=GetObject\"\n },\n {\n \"filePath\": \"test-connector/api-service/user_uploads/school_data.json\",\n \"fileName\": \"school_data.json\",\n \"preSignedUrl\": \"https://test-connector.s3.us-east-2.amazonaws.com/test-connector/api-service/user_uploads/school_data.json?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA2SANCC6IV26VYMEG%2F20240716%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20240716T040140Z&X-Amz-Expires=600&X-Amz-Signature=e02f34103615f7dcc206c3afc8365ebfe9b58a00eb4c0200aa986bce58406cbd&X-Amz-SignedHeaders=host&x-id=GetObject\"\n }\n ]\n}" - }, - { - "name": "Failure: limit exceeds", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\",\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"read\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/files/generate-url" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "355" - }, - { - "key": "ETag", - "value": "W/\"163-9oQYJJEaBH3mJAnzDHXn2MxE848\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 03:03:04 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T08:33:04+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"d3a606ca-47d0-4746-95a1-c8692e749959\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_URL_GENERATION_LIMIT_EXCEED\",\n \"message\": \"Pre-signed URL generation failed: limit exceeded.\",\n \"trace\": \"\"\n }\n}" - }, - { - "name": "Failure: Invalid request", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-19T12:58:47+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\"\n },\n \"request\": {\n \"files\": [\n \"telemetry.json\",\n \"school_data.json\"\n ],\n \"access\": \"update\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/files/generate-url" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "390" - }, - { - "key": "ETag", - "value": "W/\"186-KH9x0zC3+RqtWpa0tVT9XVg8agk\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 04:01:10 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.files.generate-url\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T09:31:10+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6\",\n \"resmsgid\": \"c3e9da1c-09f3-4a3b-84ec-a19efc68b856\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"error\": {\n \"code\": \"FILES_GENERATE_URL_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/access/enum must be equal to one of the allowed values\",\n \"trace\": \"\"\n }\n}" - } - ] - }, - { - "name": "Data schema generator", - "request": { - "method": "POST", - "header": [ - { - "key": "Accept", - "value": "application/json, text/plain, */*", - "disabled": true - }, - { - "key": "Accept-Language", - "value": "en-GB,en", - "disabled": true - }, - { - "key": "Cache-Control", - "value": "no-store", - "disabled": true - }, - { - "key": "Connection", - "value": "keep-alive", - "disabled": true - }, - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", - "disabled": true - }, - { - "key": "Origin", - "value": "http://localhost:3001", - "disabled": true - }, - { - "key": "Pragma", - "value": "no-store", - "disabled": true - }, - { - "key": "Referer", - "value": "http://localhost:3001/console/dataset/new", - "disabled": true - }, - { - "key": "Sec-Fetch-Dest", - "value": "empty", - "disabled": true - }, - { - "key": "Sec-Fetch-Mode", - "value": "cors", - "disabled": true - }, - { - "key": "Sec-Fetch-Site", - "value": "same-origin", - "disabled": true - }, - { - "key": "Sec-GPC", - "value": "1", - "disabled": true - }, - { - "key": "User-Agent", - "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "disabled": true - }, - { - "key": "sec-ch-ua", - "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", - "disabled": true - }, - { - "key": "sec-ch-ua-mobile", - "value": "?0", - "disabled": true - }, - { - "key": "sec-ch-ua-platform", - "value": "\"macOS\"", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3007/dataset/v1/dataschema" - }, - "response": [ - { - "name": "Success: Schema generated successfully", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Accept", - "value": "application/json, text/plain, */*", - "disabled": true - }, - { - "key": "Accept-Language", - "value": "en-GB,en", - "disabled": true - }, - { - "key": "Cache-Control", - "value": "no-store", - "disabled": true - }, - { - "key": "Connection", - "value": "keep-alive", - "disabled": true - }, - { - "key": "Content-Type", - "value": "application/json", - "disabled": true - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AhmHcVuLu0Xb2zekokK6sl7cZibttCOYC.k7WzCmtQ%2BgwwBV4l4Mte6nJNw4pZPHePSisPRKUeBVQ", - "disabled": true - }, - { - "key": "Origin", - "value": "http://localhost:3001", - "disabled": true - }, - { - "key": "Pragma", - "value": "no-store", - "disabled": true - }, - { - "key": "Referer", - "value": "http://localhost:3001/console/dataset/new", - "disabled": true - }, - { - "key": "Sec-Fetch-Dest", - "value": "empty", - "disabled": true - }, - { - "key": "Sec-Fetch-Mode", - "value": "cors", - "disabled": true - }, - { - "key": "Sec-Fetch-Site", - "value": "same-origin", - "disabled": true - }, - { - "key": "Sec-GPC", - "value": "1", - "disabled": true - }, - { - "key": "User-Agent", - "value": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36", - "disabled": true - }, - { - "key": "sec-ch-ua", - "value": "\"Not/A)Brand\";v=\"8\", \"Chromium\";v=\"126\", \"Brave\";v=\"126\"", - "disabled": true - }, - { - "key": "sec-ch-ua-mobile", - "value": "?0", - "disabled": true - }, - { - "key": "sec-ch-ua-platform", - "value": "\"macOS\"", - "disabled": true - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"data\": [\n {\n \"tripID\": \"b97b0dbd-1463-4a42-99ff-211fc389464c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-12 00:22:30\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:40:50\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"249\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Raquel.Kunde@gmail.com\",\n \"mobile\": \"310-255-4865 x1413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a0520d95-1ae2-4d94-a6d2-0e35857e2b3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:47:00\",\n \"tpep_dropoff_datetime\": \"2024-01-25 00:52:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Willy15@gmail.com\",\n \"mobile\": \"474-817-2801 x633\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6248522a-ffbb-4d73-8b89-75a0e5310f25\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-19 00:55:39\",\n \"tpep_dropoff_datetime\": \"2023-03-16 01:03:06\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"162\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Agustina74@yahoo.com\",\n \"mobile\": \"1-533-609-5857 x24749\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3502a658-1bf8-4d62-a180-f1560d088d36\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-02 00:28:37\",\n \"tpep_dropoff_datetime\": \"2024-01-26 00:31:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".76\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Shemar97@hotmail.com\",\n \"mobile\": \"(738) 409-8443 x5839\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"245a42b8-be3f-49a7-b82b-755a60c0fe90\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-16 00:33:19\",\n \"tpep_dropoff_datetime\": \"2023-06-17 00:46:44\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"68\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Alvis.Kshlerin77@hotmail.com\",\n \"mobile\": \"1-487-796-9469 x057\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4e351ad9-b554-49fe-bdeb-351ced4a5fbc\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-05-11 00:59:05\",\n \"tpep_dropoff_datetime\": \"2023-06-21 01:19:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"246\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Nicole.White@hotmail.com\",\n \"mobile\": \"687-578-1535 x50831\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9f24c946-4f7d-4e2b-b0f0-34c11b61b97e\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:11:27\",\n \"tpep_dropoff_datetime\": \"2024-01-14 00:46:29\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"21.42\",\n \"RatecodeID\": \"2\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"132\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Talia.Denesik@hotmail.com\",\n \"mobile\": \"727-537-1685 x107\"\n },\n \"fare_details\": {\n \"fare_amount\": \"52\",\n \"extra\": \"0\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"11.71\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"70.27\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4622df23-a8f1-482f-9310-76d6fb772b9a\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-02 00:19:27\",\n \"tpep_dropoff_datetime\": \"2023-03-06 01:03:21\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"10.84\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"114\",\n \"DOLocationID\": \"198\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alice43@gmail.com\",\n \"mobile\": \"579-402-1634 x87762\"\n },\n \"fare_details\": {\n \"fare_amount\": \"37.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"38.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"8f8211bf-9693-4b36-817f-d9c1c7253691\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-12 00:11:49\",\n \"tpep_dropoff_datetime\": \"2023-02-28 00:23:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"3.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mike88@gmail.com\",\n \"mobile\": \"(501) 617-2020 x4697\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.5\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6a102595-7db4-4184-bd80-ce249b784f0b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-02 00:38:41\",\n \"tpep_dropoff_datetime\": \"2024-02-01 01:08:52\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"3.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"249\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vella.Armstrong71@gmail.com\",\n \"mobile\": \"978.794.7934 x32048\"\n },\n \"fare_details\": {\n \"fare_amount\": \"18.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"19.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ceced196-7da9-4bac-83ad-fe0129098574\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-20 00:14:19\",\n \"tpep_dropoff_datetime\": \"2023-11-22 00:17:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".78\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Keven_Kihn@hotmail.com\",\n \"mobile\": \"1-323-467-0737 x980\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"23a467f1-de9b-474f-a60a-3d103ebeba04\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-03 00:20:57\",\n \"tpep_dropoff_datetime\": \"2023-11-12 00:30:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.71\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kiera.Kling@hotmail.com\",\n \"mobile\": \"704-373-7606 x93095\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"15c7f4d7-fa74-41ee-98d5-7c00f30bb366\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-25 00:36:13\",\n \"tpep_dropoff_datetime\": \"2023-08-19 00:46:08\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.28\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"151\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Laura_Lind99@hotmail.com\",\n \"mobile\": \"(846) 893-6673 x69711\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.16\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"796dfa5d-e16d-49c9-b41c-f4223a8fedd9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-14 00:56:36\",\n \"tpep_dropoff_datetime\": \"2023-07-08 01:04:49\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.57\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Melvin68@yahoo.com\",\n \"mobile\": \"1-436-319-7744 x1743\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.86\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.16\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b8b56888-8bb4-4981-b669-2a56d563b25f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-23 00:27:41\",\n \"tpep_dropoff_datetime\": \"2024-02-12 00:42:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mauricio8@gmail.com\",\n \"mobile\": \"1-298-756-7810 x0828\"\n },\n \"fare_details\": {\n \"fare_amount\": \"12\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4f9c0e5e-1a8d-4e47-8e02-b5d3676162f2\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-09-30 00:46:37\",\n \"tpep_dropoff_datetime\": \"2023-04-07 00:53:37\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.18\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"141\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bailee.Roob80@gmail.com\",\n \"mobile\": \"753.766.7597 x58905\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.66\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.96\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc5fb067-ee76-4c74-8d53-913518d7de3c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:21:23\",\n \"tpep_dropoff_datetime\": \"2023-08-14 00:26:07\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"233\",\n \"DOLocationID\": \"233\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Francisco70@yahoo.com\",\n \"mobile\": \"1-591-565-3358 x04096\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d1008fa4-1017-43e5-a7cd-7fcc235d82e5\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:49:45\",\n \"tpep_dropoff_datetime\": \"2023-08-30 00:56:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Charity_Jacobs58@gmail.com\",\n \"mobile\": \"1-713-793-4442 x6226\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a4d907fa-cad9-41fa-a25f-cd3fac14d0d9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-10-27 00:56:42\",\n \"tpep_dropoff_datetime\": \"2023-03-23 01:26:09\",\n \"passenger_count\": \"4\",\n \"trip_distance\": \"6.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"75\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Bridie61@yahoo.com\",\n \"mobile\": \"(632) 512-8857\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.05\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"28c84ea3-4129-4114-b0ee-7bb39cb9613f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-03-07 00:10:50\",\n \"tpep_dropoff_datetime\": \"2023-06-10 00:17:46\",\n \"passenger_count\": \"5\",\n \"trip_distance\": \"1.32\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"42\",\n \"DOLocationID\": \"41\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Vickie.Franey93@gmail.com\",\n \"mobile\": \"874.663.4243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1218ae38-7474-4a4b-9343-e3d0fe6da9c6\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-07-18 00:32:36\",\n \"tpep_dropoff_datetime\": \"2023-06-16 00:54:08\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \"3.65\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"224\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Alexandro.Prohaska40@hotmail.com\",\n \"mobile\": \"(413) 378-0852 x100\"\n },\n \"fare_details\": {\n \"fare_amount\": \"15.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a5fc580-5a28-4153-b37f-a2ecc4ebc777\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-06 00:32:24\",\n \"tpep_dropoff_datetime\": \"2023-11-29 00:33:21\",\n \"passenger_count\": \"0\",\n \"trip_distance\": \"5.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"50\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Veda_Stamm@hotmail.com\",\n \"mobile\": \"596-555-2936 x826\"\n },\n \"fare_details\": {\n \"fare_amount\": \"2.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0.75\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"4.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"89856e17-912a-40e6-8379-f1d75a066878\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-30 00:36:37\",\n \"tpep_dropoff_datetime\": \"2024-02-14 00:42:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"50\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jed60@hotmail.com\",\n \"mobile\": \"(761) 368-4573\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fece3c74-e914-46c6-8b84-88470d9c7b3a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-14 00:44:29\",\n \"tpep_dropoff_datetime\": \"2023-03-22 00:48:05\",\n \"passenger_count\": \"3\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Juwan_Schimmel@yahoo.com\",\n \"mobile\": \"1-725-659-7644\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9142ffc4-db10-4faf-ad6a-b15595e02408\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-14 00:52:53\",\n \"tpep_dropoff_datetime\": \"2023-09-29 01:27:03\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"14.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"98\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Pinkie13@hotmail.com\",\n \"mobile\": \"(991) 905-4690\"\n },\n \"fare_details\": {\n \"fare_amount\": \"42\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"49.06\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"ac62b151-e94c-473f-b461-c25f7035e8ac\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-01 00:28:12\",\n \"tpep_dropoff_datetime\": \"2023-03-08 00:34:55\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.43\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"14\",\n \"DOLocationID\": \"228\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Esteban_Ondricka@yahoo.com\",\n \"mobile\": \"(697) 411-7922\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"707e720d-afdd-4af9-9469-02fd5052ed9f\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-10-10 00:55:16\",\n \"tpep_dropoff_datetime\": \"2023-03-25 01:01:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"11\",\n \"DOLocationID\": \"22\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kristin.Baumbach-Ferry@hotmail.com\",\n \"mobile\": \"964-998-1199\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e75c5819-6f52-4c57-a6ea-85b79935821b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-11 00:29:10\",\n \"tpep_dropoff_datetime\": \"2023-08-10 01:23:09\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"11.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"181\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Dorothea.Kulas@gmail.com\",\n \"mobile\": \"354.367.7954\"\n },\n \"fare_details\": {\n \"fare_amount\": \"40\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"10.33\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"51.63\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d8fc1a3c-3c56-4f21-94d2-f42a2e5e3a9d\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-12 00:20:30\",\n \"tpep_dropoff_datetime\": \"2023-11-09 00:36:07\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"229\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jazmyn_Corwin@gmail.com\",\n \"mobile\": \"642.790.7928 x83713\"\n },\n \"fare_details\": {\n \"fare_amount\": \"10.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"6eb24260-db4a-4a2f-a2bf-fbd0cfe0ffba\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-02-04 00:42:50\",\n \"tpep_dropoff_datetime\": \"2024-03-03 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"43\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Mazie18@hotmail.com\",\n \"mobile\": \"1-641-639-4170 x249\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"18578bdf-e169-4d3d-ae08-b6320bb04e29\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-19 00:53:39\",\n \"tpep_dropoff_datetime\": \"2023-05-12 00:51:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"43\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Maud.Collins@gmail.com\",\n \"mobile\": \"214.847.9872\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.14\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"16.44\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b44c85e8-ebd2-481f-a9d5-52124dec673a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-02-25 00:16:33\",\n \"tpep_dropoff_datetime\": \"2023-09-01 00:23:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Domenick_Pagac@yahoo.com\",\n \"mobile\": \"742.839.7610 x189\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.65\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dc355724-1226-4dbf-ace7-f5c34e39b7e7\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-23 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-06-26 00:30:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"140\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Kathryne1@yahoo.com\",\n \"mobile\": \"(900) 624-9537\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e65bc848-a894-4fb7-b889-dd7d812ab793\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-01 00:34:18\",\n \"tpep_dropoff_datetime\": \"2023-10-11 00:38:47\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"140\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Hilton.Jacobi@gmail.com\",\n \"mobile\": \"1-637-451-2136\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.25\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.55\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"32bf8dee-cd36-4c9e-befb-be32256653e0\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-01 00:39:55\",\n \"tpep_dropoff_datetime\": \"2023-03-31 00:51:01\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ole.Lindgren@hotmail.com\",\n \"mobile\": \"(728) 501-8337 x72810\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"3.2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"14\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1e7bc9a0-ccc6-4855-8650-a46e8695c1a2\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:52:47\",\n \"tpep_dropoff_datetime\": \"2023-12-12 00:59:52\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"238\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jazmin.Pollich-Little81@gmail.com\",\n \"mobile\": \"517-815-1765\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b81e5976-6581-4cad-914d-b1bf3a007c7b\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-24 00:24:43\",\n \"tpep_dropoff_datetime\": \"2023-05-19 00:37:30\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"162\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jake_Heidenreich52@gmail.com\",\n \"mobile\": \"300-566-1457 x9243\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"12.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"beae8765-a627-4af7-ac5b-a5ba96f6d54f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:47:22\",\n \"tpep_dropoff_datetime\": \"2023-10-17 00:55:16\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Mitchell.Green64@hotmail.com\",\n \"mobile\": \"1-279-558-0312 x75465\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a59b934c-583c-4fa3-a5cb-15041379e8da\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-17 00:59:36\",\n \"tpep_dropoff_datetime\": \"2023-09-12 01:28:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Esther.Hintz@hotmail.com\",\n \"mobile\": \"546-446-7171 x1903\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"4.15\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.95\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"7ed9ad81-42cb-436e-93c5-d88d7245493f\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:18:04\",\n \"tpep_dropoff_datetime\": \"2023-10-04 00:25:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"137\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Carolanne_Buckridge@gmail.com\",\n \"mobile\": \"(972) 507-5995 x716\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4c325355-2689-482f-8107-ece7515ddc33\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-17 00:40:33\",\n \"tpep_dropoff_datetime\": \"2023-03-17 00:44:25\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \".40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"4\",\n \"primary_passenger\": {\n \"email\": \"Willie.Zieme44@hotmail.com\",\n \"mobile\": \"813.930.8291 x27037\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5e153af5-1e69-4acc-bd70-25b7d6e9856a\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-23 00:59:13\",\n \"tpep_dropoff_datetime\": \"2023-10-21 01:01:44\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \".50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"161\",\n \"DOLocationID\": \"161\",\n \"payment_type\": \"3\",\n \"primary_passenger\": {\n \"email\": \"Abe.Kassulke5@yahoo.com\",\n \"mobile\": \"1-994-727-5845 x8326\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"5.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"eef4af05-c391-4a67-9ed9-0a6a2c23645c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-03-11 00:10:40\",\n \"tpep_dropoff_datetime\": \"2023-03-16 00:27:11\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"4.30\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"229\",\n \"DOLocationID\": \"223\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Clifford.Fisher@yahoo.com\",\n \"mobile\": \"1-934-478-8531 x33207\"\n },\n \"fare_details\": {\n \"fare_amount\": \"16\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"a790d399-dbbc-4483-805b-0fb001b6c6b7\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-06-27 00:25:03\",\n \"tpep_dropoff_datetime\": \"2024-03-07 01:02:07\",\n \"passenger_count\": \"6\",\n \"trip_distance\": \"3.58\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"48\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Carson.Hartmann-Lynch59@gmail.com\",\n \"mobile\": \"(856) 507-5220 x738\"\n },\n \"fare_details\": {\n \"fare_amount\": \"23\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"24.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"10fb7db8-edee-4686-95e7-d07a17761fe9\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-17 00:26:54\",\n \"tpep_dropoff_datetime\": \"2023-08-06 00:49:47\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"158\",\n \"DOLocationID\": \"107\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Brandyn.Powlowski43@yahoo.com\",\n \"mobile\": \"(972) 869-2846\"\n },\n \"fare_details\": {\n \"fare_amount\": \"14.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"b575d34f-8874-4b67-8918-293cccec8558\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-03-04 00:18:00\",\n \"tpep_dropoff_datetime\": \"2023-11-02 00:26:10\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.11\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"13\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Lelia71@hotmail.com\",\n \"mobile\": \"298-955-0204 x145\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4524ce02-c41d-4e83-82a1-e41b00d97dab\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-25 00:33:25\",\n \"tpep_dropoff_datetime\": \"2023-10-29 01:07:18\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.63\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"113\",\n \"DOLocationID\": \"238\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magali_Mohr90@gmail.com\",\n \"mobile\": \"1-339-745-7996 x126\"\n },\n \"fare_details\": {\n \"fare_amount\": \"24\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.06\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"30.36\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"0a4d6094-f334-4e18-a1e9-1001820b6f89\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-26 00:11:00\",\n \"tpep_dropoff_datetime\": \"2023-08-01 00:15:28\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Sydney.Sanford71@yahoo.com\",\n \"mobile\": \"557.212.3262 x589\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.26\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"7.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"9245af5e-632e-4f21-9060-88522728ab73\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-09-07 00:17:57\",\n \"tpep_dropoff_datetime\": \"2023-09-14 00:27:43\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"3.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Deanna15@hotmail.com\",\n \"mobile\": \"927-327-6309 x16689\"\n },\n \"fare_details\": {\n \"fare_amount\": \"13\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"17.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"4dcce454-046c-4b67-bd80-ff773a3c3d96\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-16 00:35:11\",\n \"tpep_dropoff_datetime\": \"2023-05-07 00:58:40\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.40\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"261\",\n \"DOLocationID\": \"142\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Jeramie33@yahoo.com\",\n \"mobile\": \"757-419-8948 x7985\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"5.45\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"27.25\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"d5808aed-5e8e-447f-8c42-521f8472a24d\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-01-23 00:12:48\",\n \"tpep_dropoff_datetime\": \"2023-07-16 00:23:48\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"211\",\n \"DOLocationID\": \"232\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jarrod_Bergstrom@hotmail.com\",\n \"mobile\": \"388-916-2388 x911\"\n },\n \"fare_details\": {\n \"fare_amount\": \"9.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"10.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"faf99a1d-127f-432a-bdb9-39c91a205ec3\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-08-12 00:31:53\",\n \"tpep_dropoff_datetime\": \"2023-09-30 00:47:26\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"164\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"America72@gmail.com\",\n \"mobile\": \"528-291-8014 x700\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2.55\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"15.35\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"21b8ea90-d3ab-4fe6-890e-2b8a911500e1\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-05 00:54:38\",\n \"tpep_dropoff_datetime\": \"2023-08-23 01:01:13\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.00\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"107\",\n \"DOLocationID\": \"170\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Autumn_Kerluke@hotmail.com\",\n \"mobile\": \"542.726.7058\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"080446ad-cf54-429f-82e2-e54f4f8d4a9c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-16 00:00:58\",\n \"tpep_dropoff_datetime\": \"2023-10-29 00:06:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"79\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Reanna_Conroy-Ratke@gmail.com\",\n \"mobile\": \"(565) 628-3638\"\n },\n \"fare_details\": {\n \"fare_amount\": \"5.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"2\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5ec8c804-9d6c-436b-be68-da3fa2ffcc8e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-03-08 00:14:58\",\n \"tpep_dropoff_datetime\": \"2024-02-09 00:24:33\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"2.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"4\",\n \"DOLocationID\": \"87\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Randi0@yahoo.com\",\n \"mobile\": \"(248) 538-4300 x71383\"\n },\n \"fare_details\": {\n \"fare_amount\": \"11\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"13.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"3f59b9fa-e8da-4b82-ac19-3b4d5cae65e8\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-07-01 00:31:12\",\n \"tpep_dropoff_datetime\": \"2023-09-29 00:38:08\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"Y\",\n \"PULocationID\": \"148\",\n \"DOLocationID\": \"148\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Ethyl74@yahoo.com\",\n \"mobile\": \"292-960-2200 x317\"\n },\n \"fare_details\": {\n \"fare_amount\": \"6\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.8\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.1\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"dec9470a-6baa-492e-9220-d7ed626e2a99\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-22 00:43:21\",\n \"tpep_dropoff_datetime\": \"2024-03-02 00:50:49\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.10\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"79\",\n \"DOLocationID\": \"231\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Makayla_Schneider18@hotmail.com\",\n \"mobile\": \"1-885-537-0198 x47953\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"904d9b1f-1f17-4cfc-8e63-8bf57257da5e\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-29 00:54:14\",\n \"tpep_dropoff_datetime\": \"2023-05-03 01:02:32\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.50\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"231\",\n \"DOLocationID\": \"158\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Cora.Grimes@yahoo.com\",\n \"mobile\": \"843.706.7413\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.8\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"57d8ed83-afa7-44cb-a0d4-723775602c34\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2024-02-02 00:16:34\",\n \"tpep_dropoff_datetime\": \"2023-09-11 00:24:45\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.20\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"142\",\n \"DOLocationID\": \"237\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Tate.Bins@hotmail.com\",\n \"mobile\": \"(606) 682-9953 x671\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"fff2f2c8-4a30-446c-90af-443dd493886c\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-12-03 00:29:04\",\n \"tpep_dropoff_datetime\": \"2023-07-26 00:36:33\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"1.90\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"237\",\n \"DOLocationID\": \"262\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Magnus.Jacobs@hotmail.com\",\n \"mobile\": \"(955) 449-9284 x00149\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.85\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"11.15\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"004c4cc7-e809-4108-aa01-87b0ded794ad\",\n \"VendorID\": \"1\",\n \"tpep_pickup_datetime\": \"2023-05-18 00:41:00\",\n \"tpep_dropoff_datetime\": \"2023-08-05 00:59:50\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"5.80\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"244\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Palma24@gmail.com\",\n \"mobile\": \"1-861-673-8247 x142\"\n },\n \"fare_details\": {\n \"fare_amount\": \"20\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"21.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"1a01450d-537f-4f16-a2e9-f17021b077e9\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2024-01-31 00:20:53\",\n \"tpep_dropoff_datetime\": \"2023-12-21 00:40:21\",\n \"passenger_count\": \"2\",\n \"trip_distance\": \"5.70\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"164\",\n \"DOLocationID\": \"255\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Jeffry18@gmail.com\",\n \"mobile\": \"(206) 748-5730 x64895\"\n },\n \"fare_details\": {\n \"fare_amount\": \"19.5\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"5.76\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"26.56\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"5bccfd17-7813-43ca-9917-f2fe6ad49261\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-25 00:06:30\",\n \"tpep_dropoff_datetime\": \"2023-10-31 00:08:31\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \".46\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"1\",\n \"primary_passenger\": {\n \"email\": \"Leanne.Swaniawski@gmail.com\",\n \"mobile\": \"(787) 969-9302 x2684\"\n },\n \"fare_details\": {\n \"fare_amount\": \"4\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"1.59\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"6.89\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"e613b176-be28-4414-9a40-cff61e46db3b\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-08-13 00:09:35\",\n \"tpep_dropoff_datetime\": \"2023-04-01 00:17:05\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.72\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"239\",\n \"DOLocationID\": \"236\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Michel_Watsica63@yahoo.com\",\n \"mobile\": \"793.751.7397 x892\"\n },\n \"fare_details\": {\n \"fare_amount\": \"8\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"9.3\",\n \"congestion_surcharge\": \"\"\n }\n },\n {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n }\n }\n ],\n \"config\": {\n \"dataset\": \"generate-schema\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3007/dataset/v1/dataschema" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "4414" - }, - { - "key": "ETag", - "value": "W/\"113e-ykaeY2EqBHdGqGLcr7K3WSs5fYo\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 03:28:22 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"dataset.schema.identify\",\n \"ver\": \"v1\",\n \"ts\": 1721100502574,\n \"params\": {\n \"status\": \"SUCCESS\",\n \"errmsg\": \"\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"tripID\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tripID' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tripID\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"VendorID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tpep_pickup_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_pickup_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_pickup_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"tpep_dropoff_datetime\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'tpep_dropoff_datetime' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.tpep_dropoff_datetime\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"passenger_count\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"trip_distance\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"RatecodeID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"store_and_fwd_flag\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"PULocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"DOLocationID\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"payment_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"primary_passenger\": {\n \"type\": \"object\",\n \"properties\": {\n \"email\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mobile\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"fare_details\": {\n \"type\": \"object\",\n \"properties\": {\n \"fare_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"extra\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mta_tax\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tip_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"tolls_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"improvement_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"total_amount\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"congestion_surcharge\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 98,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" - } - ] - }, - { - "name": "Dataset update", - "request": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721135455988\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/update" - }, - "response": [ - { - "name": "Success: Minimal dataset update", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/update" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "331" - }, - { - "key": "ETag", - "value": "W/\"14b-fNmMHDpT4Ka5pwuzbYvZo7jECEo\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 13:00:45 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:30:45+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"354f1fec-0c39-42ee-a52a-49552f847c11\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134845559\"\n }\n}" - }, - { - "name": "Success: Updated successfully", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/update" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "331" - }, - { - "key": "ETag", - "value": "W/\"14b-y8oEEJijvIDh8wU5ogipKdkv8y0\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 12:57:55 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:27:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"6d835f07-aed5-4e8b-81c2-2142cfb55c52\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"message\": \"Dataset is updated successfully\",\n \"id\": \"telemetry_record-t4\",\n \"version_key\": \"1721134675878\"\n }\n}" - }, - { - "name": "Failure: Outdated key provided", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t4\",\n \"version_key\": \"1721064642580\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/update" - }, - "status": "Conflict", - "code": 409, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "383" - }, - { - "key": "ETag", - "value": "W/\"17f-JnlFVLXyuhwx9KbxYWDRB4mmvVw\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 12:53:16 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:23:16+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fe03f6-c4c4-48f6-9d84-a32cd52f4c13\"\n },\n \"responseCode\": \"CONFLICT\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_OUTDATED\",\n \"message\": \"The dataset is outdated. Please try to fetch latest changes of the dataset and perform the updates\"\n }\n}" - }, - { - "name": "Failure: Dataset not exists to update", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/update" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "340" - }, - { - "key": "ETag", - "value": "W/\"154-4I5VyTBINyYBZZM8Ge9Cnqz2xBY\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 12:58:30 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:28:30+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf64703c-bb6b-41bf-bc1a-c85373efd925\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_EXISTS\",\n \"message\": \"Dataset does not exists with id:telemetry_record-t41\"\n }\n}" - }, - { - "name": "Failure: Invalid request", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\",\n \"name\": \"sb-telemetry\",\n \"validation_config\": {\n \"validate\": true,\n \"mode\": \"Strict\"\n },\n \"extraction_config\": {\n \"is_batch_event\": true,\n \"extraction_key\": \"events\",\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"ipid\"\n }\n },\n \"dedup_config\": {\n \"drop_duplicates\": true,\n \"dedup_key\": \"mid\"\n },\n \"data_schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"midpid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"miduwi\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"additionalProperties\": true\n },\n \"denorm_config\": {\n \"denorm_fields\": [\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"userdata\"\n \n },\n \"action\": \"remove\"\n },\n {\n \"value\": {\n \"denorm_key\": \"eid\",\n \"denorm_out_field\": \"edata\",\n \"dataset_id\": \"trip-details\"\n },\n \"action\": \"upsert\"\n }\n ]\n },\n \"transformations_config\": [\n {\n \"value\": {\n \"field_key\": \"email\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"upsert\"\n },\n {\n \"value\": {\n \"field_key\": \"email_id\",\n \"transformation_function\": {\n \"type\": \"mask\",\n \"expr\": \"mid\",\n \"datatype\":\"string\",\n \"category\":\"pii\"\n },\n \"mode\": \"Strict\"\n },\n \"action\": \"remove\"\n }\n ],\n \"tags\": [\n \n ],\n \"connectors_config\":[\n {\"value\":{\n \n \"connector_id\": \"jdbc\",\n \"connector_config\": {\n \"source_database_type\": \"postgresql\",\n \"source_database_host\": \"postgresql-hl.postgresql.svc.cluster.local.master\",\n \"source_database_port\": 5432,\n \"source_database_name\": \"obsrv_sample_datasets_1\",\n \"source_database_username\": \"postgres\",\n \"source_database_pwd\": \"postgres\",\n \"table\": \"new_york_taxi_data\",\n \"timestamp-column\": \"tpep_pickup_datetime\",\n \"batch-size\": 100,\n \"max-batches\": 2\n },\n \"operations_config\": {\n \"polling_interval\": \"periodic\",\n \"schedule\": \"twice\"\n }\n \n }, \"action\":\"upsert\"}\n ]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "369" - }, - { - "key": "ETag", - "value": "W/\"171-iNJoyWUecOEsXbHZwx6rld3Sr1I\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 12:59:21 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:29:21+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"7d31672b-e5c3-4a6d-afac-d9d78011bcde\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_INPUT_INVALID\",\n \"message\": \"#properties/request/required must have required property 'dataset_id'\"\n }\n}" - }, - { - "name": "Failure: No fields are provided to update", - "originalRequest": { - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"dataset_id\": \"telemetry_record-t41\",\n \"version_key\": \"1721049248930\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/update" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "373" - }, - { - "key": "ETag", - "value": "W/\"175-ga30XLi7qSW4Ix+3Q/xaMUYcqII\"" - }, - { - "key": "Date", - "value": "Tue, 16 Jul 2024 13:02:44 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.update\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-16T18:32:44+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bf99b1e1-7694-4be0-ba5d-e347764736de\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_UPDATE_NO_FIELDS\",\n \"message\": \"Provide atleast one field in addition to the dataset_id to update the dataset\"\n }\n}" - } - ] - }, - { - "name": "Read dataset", - "request": { - "method": "GET", - "header": [ - { - "key": "Cookie", - "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" - } - ], - "url": { - "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", - "host": [ - "localhost" - ], - "port": "3007", - "path": [ - "v2", - "datasets", - "read", - "beckn-test-data" - ], - "query": [ - { - "key": "mode", - "value": "edit" - } - ] - } - }, - "response": [ - { - "name": "Success: Read live dataset", - "originalRequest": { - "method": "GET", - "header": [ - { - "key": "Cookie", - "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" - } - ], - "url": "localhost:3007/v2/datasets/read/master-test" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "501" - }, - { - "key": "ETag", - "value": "W/\"1f5-p+b/6r0nHRFhgr5+URzxk4d/CSg\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:08:55 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:38:55+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"8c8a2852-54bc-43fb-b063-7f359d11930a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n }\n}" - }, - { - "name": "Success: Read draft dataset", - "originalRequest": { - "method": "GET", - "header": [ - { - "key": "Cookie", - "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" - } - ], - "url": { - "raw": "localhost:3007/v2/datasets/read/beckn-test-data?mode=edit", - "host": [ - "localhost" - ], - "port": "3007", - "path": [ - "v2", - "datasets", - "read", - "beckn-test-data" - ], - "query": [ - { - "key": "mode", - "value": "edit" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "528" - }, - { - "key": "ETag", - "value": "W/\"210-Qo8q3dU8l7LYIXVzJwStVe90z9Q\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:11:00 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:41:00+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"96fd4f42-fa84-4730-bc79-d241a4e335a1\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n }\n}" - }, - { - "name": "Success: Read specific column", - "originalRequest": { - "method": "GET", - "header": [ - { - "key": "Cookie", - "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" - } - ], - "url": { - "raw": "localhost:3007/v2/datasets/read/master-test?status=Draft&fields=name,type,id", - "host": [ - "localhost" - ], - "port": "3007", - "path": [ - "v2", - "datasets", - "read", - "master-test" - ], - "query": [ - { - "key": "status", - "value": "Draft" - }, - { - "key": "fields", - "value": "name,type,id" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "237" - }, - { - "key": "ETag", - "value": "W/\"ed-zvknH4AY6kid9Yit+KqMIdDeNGc\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:12:16 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:42:16+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"02a6b03a-8bf3-4e37-8dcd-859d3e8f904e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"id\": \"master-test\"\n }\n}" - }, - { - "name": "Success: Read version_key", - "originalRequest": { - "method": "GET", - "header": [ - { - "key": "Cookie", - "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" - } - ], - "url": { - "raw": "localhost:3007/v2/datasets/read/beckn-test-data?fields=version_key&mode=edit", - "host": [ - "localhost" - ], - "port": "3007", - "path": [ - "v2", - "datasets", - "read", - "beckn-test-data" - ], - "query": [ - { - "key": "fields", - "value": "version_key" - }, - { - "key": "mode", - "value": "edit" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "211" - }, - { - "key": "ETag", - "value": "W/\"d3-hfyvp0pFPZhM2NbiiMDTfy+51YM\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:15:37 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:45:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"805e848a-d260-47c3-b55c-fc9b8323719e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"version_key\": \"1718791650227\"\n }\n}" - }, - { - "name": "Success: Read from draft, if not present cerate draft from live dataset and then read", - "originalRequest": { - "method": "GET", - "header": [ - { - "key": "Cookie", - "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" - } - ], - "url": { - "raw": "localhost:3007/v2/datasets/read/sample1?mode=edit", - "host": [ - "localhost" - ], - "port": "3007", - "path": [ - "v2", - "datasets", - "read", - "sample1" - ], - "query": [ - { - "key": "mode", - "value": "edit" - } - ] - } - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "538" - }, - { - "key": "ETag", - "value": "W/\"21a-Iqg1rOMRqVpT0n8ZhQsPiTB2l44\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:19:28 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:49:28+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"da70e25b-6ad0-49a7-a39d-340d1d0c46a7\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 2,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": false,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\"\n },\n \"cache_config\": {\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"redis_db\": 0\n }\n }\n }\n}" - }, - { - "name": "Failure: Invalid field name provided", - "originalRequest": { - "method": "GET", - "header": [ - { - "key": "Cookie", - "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" - } - ], - "url": { - "raw": "localhost:3007/v2/datasets/read/telemetry_record?fields=newname", - "host": [ - "localhost" - ], - "port": "3007", - "path": [ - "v2", - "datasets", - "read", - "telemetry_record" - ], - "query": [ - { - "key": "fields", - "value": "newname" - } - ] - } - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "307" - }, - { - "key": "ETag", - "value": "W/\"133-TQ9WpmutsrDcTNkRRmbWOhUChMk\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:20:17 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:50:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"bccd40ad-db0a-4ed5-984c-e89a9d7b3fdd\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_FIELDS\",\n \"message\": \"The specified fields [newname] in the dataset cannot be found.\"\n }\n}" - }, - { - "name": "Failure: Dataset not found", - "originalRequest": { - "method": "GET", - "header": [ - { - "key": "Cookie", - "value": "connect.sid=s%3AAYYroI28UhzQVPM909UpLjZlcqMlDMlZ.gAO6bTMTktZi7udh7jntL%2Bw2xVWiI1z6gsSAb3bhZp4" - } - ], - "url": "localhost:3007/v2/datasets/read/new_telemetry_record.1" - }, - "status": "Not Found", - "code": 404, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "304" - }, - { - "key": "ETag", - "value": "W/\"130-JL/oBB+GUHTrBp278giBHRvO71I\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:21:12 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.read\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:51:12+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"3aad3842-a76e-4fe8-b635-c7fef5f318f9\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_NOT_FOUND\",\n \"message\": \"Dataset with the given dataset_id:new_telemetry_record.1 not found\"\n }\n}" - } - ] - }, - { - "name": "Dataset list", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/list" - }, - "response": [ - { - "name": "Success: Lists all when no filters provided", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "24340" - }, - { - "key": "ETag", - "value": "W/\"5f14-Cq3tfdk3YuXhXtjub1V0q8YVdC4\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:25:36 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:55:36+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"97efe04d-e981-493d-9ee7-a6dad6887d64\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry_record-t4\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_events\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"telemetry_record-master\",\n \"name\": \"sb-telemetry\",\n \"type\": \"master\",\n \"status\": \"Draft\",\n \"tags\": [\n \"tag1\"\n ],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"indexing_config\": {\n \"olap_store_enabled\": true,\n \"lakehouse_enabled\": true,\n \"cache_enabled\": false\n },\n \"keys_config\": {\n \"data_key\": \"\",\n \"partition_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\"\n },\n \"cache_config\": {\n \"redis_db_port\": null,\n \"redis_db\": 0\n },\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"generate-schema\",\n \"name\": \"generate-schema\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-test\",\n \"name\": \"trip-test\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample-trip-details\",\n \"name\": \"sample-trip-details\",\n \"type\": \"event\",\n \"status\": \"Draft\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 30\n }\n}" - }, - { - "name": "Success: Filter based on status as array", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Live\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "5475" - }, - { - "key": "ETag", - "value": "W/\"1563-FnaUnwhv4LmMcE5J8f4+jKz+DDk\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:27:38 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:57:38+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"31aba5bc-8492-45ce-be0e-8c52d8716014\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"telemetry-summary\",\n \"name\": \"telemetry-summary\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"tripdetailstest\",\n \"name\": \"TripDetailsTest1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-normal\",\n \"name\": \"test-normal-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-dataset\",\n \"name\": \"test-dataset-renamed\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_dropoff_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"triptestdataset\",\n \"name\": \"triptestdataset\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"test-trip-details\",\n \"name\": \"test-trip-details\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": []\n }\n },\n {\n \"dataset_id\": \"sb-telemetry\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-test\",\n \"name\": \"sb-telemetry\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-changes\",\n \"name\": \"test-changes\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"sample1\",\n \"name\": \"sample1\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"time\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-events\",\n \"name\": \"telemetry-events\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"taxt_trip\",\n \"name\": \"taxt_trip\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"date\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test\",\n \"name\": \"test\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [\n \"TAG1\"\n ],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"exclude_fields\": [],\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n }\n ],\n \"count\": 16\n }\n}" - }, - { - "name": "Success: Filter basen on status as string", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"ReadyToPublish\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "16960" - }, - { - "key": "ETag", - "value": "W/\"4240-18LMqkfsWst/M+Uc53td59PATTk\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:29:18 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T17:59:18+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"a08c7ea0-bb1c-4998-b47d-a76e38e87e31\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"test-summary\",\n \"name\": \"test-summary\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"trip-details1\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"tpep_pickup_datetime\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"telemetry-test-dataset\",\n \"name\": \"telemetry-test-dataset\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": null,\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0\n }\n },\n {\n \"dataset_id\": \"test-rollup\",\n \"name\": \"test-rollup\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {}\n },\n \"processing\": {\n \"dedupKeys\": [],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"model\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"granularity\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"date_range\": {\n \"type\": \"object\",\n \"properties\": {\n \"from\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.from' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.from\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"to\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.date_range.to' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.context.properties.date_range.properties.to\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"rollup\": {\n \"type\": \"object\",\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"dimensions\": {\n \"type\": \"object\",\n \"properties\": {\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"sid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mode\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"content_type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"eks\": {\n \"type\": \"object\",\n \"properties\": {\n \"interact_events_per_min\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"start_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.start_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.start_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"interact_events_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"item_responses\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"end_time\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'edata.eks.end_time' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.eks.properties.end_time\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"events_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"page_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"visit_count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_diff\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n },\n \"telemetry_version\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"env_summary\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"time_spent\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n },\n \"count\": {\n \"type\": \"integer\",\n \"arrival_format\": \"number\",\n \"data_type\": \"integer\"\n }\n },\n \"additionalProperties\": true\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"time_spent\": {\n \"type\": \"number\",\n \"arrival_format\": \"number\",\n \"data_type\": \"number\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"object\": {\n \"type\": \"object\",\n \"properties\": {\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\",\n \"additionalProperties\": true\n }\n },\n \"additionalProperties\": true\n }\n }\n },\n {\n \"dataset_id\": \"trip\",\n \"name\": \"trip\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"test1\",\n \"name\": \"test1\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n },\n {\n \"dataset_id\": \"beckn-test-data\",\n \"name\": \"beckn-test-data\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"ets\",\n \"entry_topic\": \"beckn-test-data\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"exclude_fields\": []\n }\n },\n {\n \"dataset_id\": \"trip-details\",\n \"name\": \"trip-details\",\n \"type\": \"event\",\n \"status\": \"ReadyToPublish\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v2\",\n \"dataset_config\": {\n \"data_key\": \"\",\n \"timestamp_key\": \"obsrv_meta.syncts\",\n \"entry_topic\": \"local.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 0,\n \"file_upload_path\": [],\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"tripID\": {\n \"path\": \"$.tripID\",\n \"cardinality\": 99,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"tripID\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"mergedEvent\": {\n \"tripID\": \"02e07922-e8a5-4655-84a8-b5ba1866f9fe\",\n \"VendorID\": \"2\",\n \"tpep_pickup_datetime\": \"2023-04-28 00:18:42\",\n \"tpep_dropoff_datetime\": \"2024-02-15 00:24:38\",\n \"passenger_count\": \"1\",\n \"trip_distance\": \"1.60\",\n \"RatecodeID\": \"1\",\n \"store_and_fwd_flag\": \"N\",\n \"PULocationID\": \"236\",\n \"DOLocationID\": \"239\",\n \"payment_type\": \"2\",\n \"primary_passenger\": {\n \"email\": \"Dewayne_Kuvalis17@gmail.com\",\n \"mobile\": \"1-429-628-3797 x14211\"\n },\n \"fare_details\": {\n \"fare_amount\": \"7\",\n \"extra\": \"0.5\",\n \"mta_tax\": \"0.5\",\n \"tip_amount\": \"0\",\n \"tolls_amount\": \"0\",\n \"improvement_surcharge\": \"0.3\",\n \"total_amount\": \"8.3\",\n \"congestion_surcharge\": \"\"\n },\n \"passenger-name\": \"yashashk\"\n }\n }\n }\n ],\n \"count\": 8\n }\n}" - }, - { - "name": "Success: Filter based on dataset type", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": \"Live\",\n \"type\": \"master\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "933" - }, - { - "key": "ETag", - "value": "W/\"3a5-IM9o2EpRUzccL+l2CW/BBpoBMqA\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:30:41 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:00:41+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"0d1ff2de-42c9-4192-b75d-84f711dbfb55\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"dataset_id\": \"master-test\",\n \"name\": \"master-test\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"userid\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"local.masterdata.ingest\",\n \"redis_db_host\": \"localhost\",\n \"redis_db_port\": 6379,\n \"index_data\": true,\n \"redis_db\": 54\n }\n },\n {\n \"dataset_id\": \"sb-telemetry-user\",\n \"name\": \"sb-telemetry-user\",\n \"type\": \"master\",\n \"status\": \"Live\",\n \"tags\": [],\n \"version\": 1,\n \"api_version\": \"v1\",\n \"dataset_config\": {\n \"data_key\": \"id\",\n \"timestamp_key\": \"\",\n \"exclude_fields\": [],\n \"entry_topic\": \"sb-dev.masterdata.ingest\",\n \"redis_db_host\": \"obsrv-redis-master.redis.svc.cluster.local\",\n \"redis_db_port\": 6379,\n \"index_data\": false,\n \"redis_db\": 4\n }\n }\n ],\n \"count\": 2\n }\n}" - }, - { - "name": "Failure: Invalid payload", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"mid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"type\": \"nodataset\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/list" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "312" - }, - { - "key": "ETag", - "value": "W/\"138-XQplwhrgIYKIg0qtQdRCYWIGTNM\"" - }, - { - "key": "Date", - "value": "Wed, 17 Jul 2024 12:32:26 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.list\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-17T18:02:26+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"add9dbe0-f362-4f99-890c-3387c998a049\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_LIST_INPUT_INVALID\",\n \"message\": \"#properties/params/required must have required property 'msgid'\"\n }\n}" - } - ] - }, - { - "name": "Data schema generator", - "request": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ],\n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/dataschema" - }, - "response": [ - { - "name": "Data schema generated successfully", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ],\n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/dataschema" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "10760" - }, - { - "key": "ETag", - "value": "W/\"2a08-QF5x1q0kIlfE9XU/pa9IboJuY8I\"" - }, - { - "key": "Date", - "value": "Mon, 22 Jul 2024 07:02:50 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:32:50+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1309aea0-9a97-46e9-bc5e-a16a8a7fb624\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"schema\": {\n \"$schema\": \"https://json-schema.org/draft/2020-12/schema\",\n \"type\": \"object\",\n \"properties\": {\n \"eid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ets\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'ets' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.ets\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"mid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'mid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.mid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"actor\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'actor.id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.actor.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"context\": {\n \"type\": \"object\",\n \"properties\": {\n \"channel\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pdata\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"ver\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pid\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"env\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"sid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'sid'. The property sid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.sid\"\n },\n {\n \"message\": \"The Property 'context.sid' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.sid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"did\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"cdata\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"object\",\n \"properties\": {\n \"id\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"The Property 'context.cdata[*].id' appears to be 'uuid' format type.\",\n \"advice\": \"Suggest to not to index the high cardinal columns\",\n \"resolutionType\": \"DEDUP\",\n \"severity\": \"LOW\",\n \"path\": \"properties.context.properties.cdata.items.properties.id\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n }\n },\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"rollup\": {\n \"type\": \"object\",\n \"properties\": {\n \"l1\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'l1'. The property l1: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.rollup.properties.l1\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"uid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uid'. The property uid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.context.properties.uid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"object\": {\n \"type\": \"object\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'object'. The property object: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.object\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"tags\": {\n \"type\": \"array\",\n \"items\": {\n \"type\": \"string\"\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'tags'. The property tags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.tags\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"edata\": {\n \"type\": \"object\",\n \"properties\": {\n \"type\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"pageid\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'pageid'. The property pageid: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.pageid\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"subtype\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'subtype'. The property subtype: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.subtype\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"uri\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'uri'. The property uri: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.uri\"\n },\n {\n \"message\": \"The Property 'edata.uri' appears to be 'uri' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.edata.properties.uri\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"visits\": {\n \"type\": \"array\",\n \"items\": false,\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'visits'. The property visits: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.edata.properties.visits\"\n }\n ],\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n },\n \"level\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"message\": {\n \"type\": \"string\",\n \"arrival_format\": \"text\",\n \"data_type\": \"string\"\n },\n \"params\": {\n \"type\": \"array\",\n \"items\": false,\n \"arrival_format\": \"array\",\n \"data_type\": \"array\"\n }\n },\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n },\n \"syncts\": {\n \"type\": \"integer\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'syncts'. The property syncts: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.syncts\"\n },\n {\n \"message\": \"The Property 'syncts' appears to be 'epoch' format type.\",\n \"severity\": \"\",\n \"path\": \"properties.syncts\"\n }\n ],\n \"arrival_format\": \"number\",\n \"data_type\": \"epoch\"\n },\n \"@timestamp\": {\n \"type\": \"string\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: '@timestamp'. The property @timestamp: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.@timestamp\"\n },\n {\n \"message\": \"The Property '@timestamp' appears to be 'date-time' format type.\",\n \"advice\": \"The System can index all data on this column\",\n \"resolutionType\": \"INDEX\",\n \"severity\": \"LOW\",\n \"path\": \"properties.@timestamp\"\n }\n ],\n \"arrival_format\": \"text\",\n \"data_type\": \"date-time\"\n },\n \"flags\": {\n \"type\": \"object\",\n \"properties\": {\n \"ex_processed\": {\n \"type\": \"boolean\",\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'ex_processed'. The property ex_processed: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags.properties.ex_processed\"\n }\n ],\n \"arrival_format\": \"boolean\",\n \"data_type\": \"boolean\"\n }\n },\n \"suggestions\": [\n {\n \"message\": \"Conflict in the Schema Generation at property: 'flags'. The property flags: only 1 time(s) appeared \",\n \"advice\": \"The Property looks to be Optional. System has updated the property schema to optional\",\n \"resolutionType\": \"OPTIONAL\",\n \"severity\": \"MEDIUM\",\n \"path\": \"properties.flags\"\n }\n ],\n \"arrival_format\": \"object\",\n \"data_type\": \"object\"\n }\n },\n \"additionalProperties\": true\n },\n \"configurations\": {\n \"indexConfiguration\": {\n \"index\": {\n \"Event Arrival Time\": \"obsrv_meta.syncts\"\n },\n \"rollupSuggestions\": {\n \"summary\": {\n \"mid\": {\n \"path\": \"$.mid\",\n \"cardinality\": 67,\n \"index\": false\n },\n \"actor.id\": {\n \"path\": \"$.actor.properties.id\",\n \"cardinality\": 56,\n \"index\": false\n },\n \"context.sid\": {\n \"path\": \"$.context.properties.sid\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"edata.uri\": {\n \"path\": \"$.edata.properties.uri\",\n \"cardinality\": 11,\n \"index\": true\n },\n \"context.cdata[*].id\": {\n \"path\": \"$.context.properties.cdata.items.properties.id\",\n \"cardinality\": 62,\n \"index\": false\n }\n }\n }\n },\n \"processing\": {\n \"dedupKeys\": [\n \"mid\",\n \"context.cdata[*].id\",\n \"actor.id\"\n ],\n \"dropDuplicates\": [\n \"Yes\",\n \"No\"\n ]\n }\n },\n \"dataMappings\": {\n \"text\": {\n \"arrival_format\": [\n \"string\"\n ],\n \"store_format\": {\n \"string\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date-time\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"date\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"string\"\n },\n \"boolean\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"boolean\"\n },\n \"epoch\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"integer\"\n },\n \"long\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"double\"\n },\n \"integer\": {\n \"jsonSchema\": \"string\",\n \"datasource\": \"long\"\n }\n }\n },\n \"number\": {\n \"arrival_format\": [\n \"number\",\n \"integer\"\n ],\n \"store_format\": {\n \"integer\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"float\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"long\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"double\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"bigdecimal\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n },\n \"epoch\": {\n \"jsonSchema\": \"integer\",\n \"datasource\": \"long\"\n },\n \"number\": {\n \"jsonSchema\": \"number\",\n \"datasource\": \"double\"\n }\n }\n },\n \"object\": {\n \"arrival_format\": [\n \"object\"\n ],\n \"store_format\": {\n \"object\": {\n \"jsonSchema\": \"object\",\n \"datasource\": \"json\"\n }\n }\n },\n \"array\": {\n \"arrival_format\": [\n \"array\"\n ],\n \"store_format\": {\n \"array\": {\n \"jsonSchema\": \"array\",\n \"datasource\": \"array\"\n }\n }\n },\n \"boolean\": {\n \"arrival_format\": [\n \"boolean\"\n ],\n \"store_format\": {\n \"boolean\": {\n \"jsonSchema\": \"boolean\",\n \"datasource\": \"boolean\"\n }\n }\n }\n }\n }\n}" - }, - { - "name": "Failure: Invalid request body", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \n \"config\": {\n \"dataset\": \"financial_transactions\"\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/dataschema" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "364" - }, - { - "key": "ETag", - "value": "W/\"16c-tfKVtCWTjNkWCtH8cFw1RrzbgV0\"" - }, - { - "key": "Date", - "value": "Mon, 22 Jul 2024 07:03:47 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:33:47+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"bbcc86c2-042d-4f77-bb6e-e1c9116df570\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'data'\"\n }\n}" - }, - { - "name": "Failure: Invalid request (config not provided)", - "originalRequest": { - "method": "POST", - "header": [ - { - "key": "Content-Type", - "value": "application/json" - }, - { - "key": "Cookie", - "value": "connect.sid=s%3AJzaMWaCpHb1z3bsRWPA9oP7-CQ0SrTch.0WR3PbOYcF4NXk4I6cTfvM1o%2F7Hq5x%2BekUOnwguHHHA" - } - ], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"data\": [\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1.672657002221E12,\n \"ver\": \"3.0\",\n \"mid\": \"IMPRESSION:2b5834e196f485c17c4e49d292af43c0\",\n \"actor\": {\n \"id\": \"0c45959486f579c24854d40a225d6161\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"01268904781886259221\",\n \"pdata\": {\n \"id\": \"staging.diksha.portal\",\n \"ver\": \"5.1.0\",\n \"pid\": \"sunbird-portal\"\n },\n \"env\": \"public\",\n \"sid\": \"23850c90-8a8c-11ed-95d0-276800e1048c\",\n \"did\": \"0c45959486f579c24854d40a225d6161\",\n \"cdata\": [],\n \"rollup\": {\n \"l1\": \"01268904781886259221\"\n },\n \"uid\": \"anonymous\"\n },\n \"object\": {},\n \"tags\": [\n \"01268904781886259221\"\n ],\n \"edata\": {\n \"type\": \"view\",\n \"pageid\": \"login\",\n \"subtype\": \"pageexit\",\n \"uri\": \"https://staging.sunbirded.org/auth/realms/sunbird/protocol/openid-connect/auth?client_id\\u003dportal\\u0026state\\u003d254efd70-6b89-4f7d-868b-5c957f54174e\\u0026redirect_uri\\u003dhttps%253A%252F%252Fstaging.sunbirded.org%252Fresources%253Fboard%253DState%252520(Andhra%252520Pradesh)%2526medium%253DEnglish%2526gradeLevel%253DClass%2525201%2526%2526id%253Dap_k-12_1%2526selectedTab%253Dhome%2526auth_callback%253D1\\u0026scope\\u003dopenid\\u0026response_type\\u003dcode\\u0026version\\u003d4\",\n \"visits\": []\n },\n \"syncts\": 1672657005814,\n \"@timestamp\": \"2023-01-02T10:56:45.814Z\",\n \"flags\": {\n \"ex_processed\": true\n }\n },\n {\n \"eid\": \"IMPRESSION\",\n \"ets\": 1672656997928,\n \"ver\": \"3.0\",\n \"mid\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"50263f0f-c2d5-4b15-95f4-5384c537f6cc\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"POST\"\n },\n {\n \"url\": \"/v1/org/search\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672656998024,\n \"ver\": \"3.0\",\n \"mid\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"4a340ad0-0665-49b6-a1fa-a581dcac4550\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=4a340ad0-0665-49b6-a1fa-a581dcac4550, type=system, message=EXIT LOG: method : POST, url: /v1/org/search , For Operation : orgSearch, params=[{msgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, errmsg=Invalid value null for parameter hashTagId. Please provide a valid value., resmsgid=4a340ad0-0665-49b6-a1fa-a581dcac4550, err=UOS_ORGSER0017, status=FAILED, responseCode=400}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657004961,\n \"ver\": \"3.0\",\n \"mid\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"actor\": {\n \"id\": \"internal\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"Organisation\",\n \"cdata\": [\n {\n \"id\": \"f34112c7242a3e3a26f0015796b029c2\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"ElasticSearchRestHighImpl:search: calling search for index org_alias, with query = {\\\"from\\\":0,\\\"size\\\":250,\\\"query\\\":{\\\"bool\\\":{\\\"must\\\":[{\\\"term\\\":{\\\"isTenant.raw\\\":{\\\"value\\\":true,\\\"boost\\\":1.0}}},{\\\"term\\\":{\\\"slug.raw\\\":{\\\"value\\\":\\\"ntp\\\",\\\"boost\\\":1.0}}}],\\\"adjust_pure_negative\\\":true,\\\"boost\\\":1.0}},\\\"_source\\\":{\\\"includes\\\":[],\\\"excludes\\\":[]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006595,\n \"ver\": \"3.0\",\n \"mid\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"actor\": {\n \"id\": \"930a3994-cbe7-4e84-936f-4974096af6f2\",\n \"type\": \"Consumer\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"d23ff123-40f0-4262-a69b-b75b46d315a1\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=d23ff123-40f0-4262-a69b-b75b46d315a1, type=system, message=ENTRY LOG: method : GET, url: /v1/user/role/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserRolesById, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006611,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"{eid='LOG', edata={level=trace, requestid=7d944b1c-a906-4082-b42a-905aa6b78a4e, type=system, message=ENTRY LOG: method : GET, url: /v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859 , For Operation : getUserProfileV5, params=[{id=null, userId=6ab35eea-01fd-4de0-8902-f68722caf859}]}}\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006620,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user_roles WHERE userId=?;\",\n \"params\": []\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657006645,\n \"ver\": \"3.0\",\n \"mid\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.learning.service\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"cdata\": [\n {\n \"id\": \"7d944b1c-a906-4082-b42a-905aa6b78a4e\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"\",\n \"params\": [\n {\n \"method\": \"GET\"\n },\n {\n \"url\": \"/v5/user/read/6ab35eea-01fd-4de0-8902-f68722caf859\"\n },\n {\n \"duration\": 0\n },\n {\n \"status\": \"OK\"\n }\n ]\n }\n },\n {\n \"eid\": \"LOG\",\n \"ets\": 1672657007238,\n \"ver\": \"3.0\",\n \"mid\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"actor\": {\n \"id\": \"6ab35eea-01fd-4de0-8902-f68722caf859\",\n \"type\": \"User\"\n },\n \"context\": {\n \"channel\": \"0126796199493140480\",\n \"pdata\": {\n \"id\": \"staging.sunbird.portal\",\n \"pid\": \"learner-service\",\n \"ver\": \"5.0.0\"\n },\n \"env\": \"User\",\n \"did\": \"d904c90d9f81ddac20141b94ddd606a0\",\n \"cdata\": [\n {\n \"id\": \"d4d34fde-c407-efb6-03bd-9f892ca0f114\",\n \"type\": \"Request\"\n }\n ],\n \"rollup\": {}\n },\n \"edata\": {\n \"level\": \"info\",\n \"type\": \"Api_access\",\n \"message\": \"Cassandra query : SELECT * FROM sunbird.user WHERE id=?;\",\n \"params\": []\n }\n }\n ]\n \n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "localhost:3007/v2/datasets/dataschema" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "366" - }, - { - "key": "ETag", - "value": "W/\"16e-YeX++2sGUWsjHqjP2GoTgqujU+c\"" - }, - { - "key": "Date", - "value": "Mon, 22 Jul 2024 07:05:36 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.datasets.dataschema\",\n \"ver\": \"v1\",\n \"ts\": \"2024-07-22T12:35:36+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"1f856c5e-37f0-41e9-96fb-642471228da2\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATA_SCHEMA_INVALID_INPUT\",\n \"message\": \"#properties/request/required must have required property 'config'\"\n }\n}" - } - ] - } - ] - }, - { - "name": "Connector api's", - "item": [ - { - "name": "Connector list", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3000/v2/connectors/list" - }, - "response": [ - { - "name": "Success: Filtered based on category", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6dc\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3000/v2/connectors/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "1908" - }, - { - "key": "ETag", - "value": "W/\"774-58gfxDsxY66h3KZnTg62QbkxcPY\"" - }, - { - "key": "Date", - "value": "Tue, 30 Jul 2024 09:37:42 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:07:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"abc\",\n \"resmsgid\": \"632d3342-fd8a-47f7-afbb-96402a00b92f\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 3\n }\n}" - }, - { - "name": "Failure: Invalid request body, filter option array should not be empty", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3000/v2/connectors/list" - }, - "status": "Bad Request", - "code": 400, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "399" - }, - { - "key": "ETag", - "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" - }, - { - "key": "Date", - "value": "Tue, 30 Jul 2024 09:43:14 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" - }, - { - "name": "Success: Connectors list with all filter options", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3000/v2/connectors/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "4107" - }, - { - "key": "ETag", - "value": "W/\"100b-SEJt8xn1TwmZz8DKpFAwkYx6E8s\"" - }, - { - "key": "Date", - "value": "Tue, 30 Jul 2024 09:47:51 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:17:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"fffa3ee0-da12-4bea-9b72-365571a62a4e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" - }, - { - "name": "Success: Connectors list without filters", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3000/v2/connectors/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "4107" - }, - { - "key": "ETag", - "value": "W/\"100b-N8mD4H5+SlIOmnNcq6Fnlq2pwXs\"" - }, - { - "key": "Date", - "value": "Tue, 30 Jul 2024 09:50:11 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:20:11+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c527a46-7bfe-4c74-a290-4dcf1a3c5f10\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" - }, - { - "name": "Success: Filtered based on status", - "originalRequest": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": "http://localhost:3000/v2/connectors/list" - }, - "status": "OK", - "code": 200, - "_postman_previewlanguage": "json", - "header": [ - { - "key": "X-Powered-By", - "value": "Express" - }, - { - "key": "Content-Type", - "value": "application/json; charset=utf-8" - }, - { - "key": "Content-Length", - "value": "250" - }, - { - "key": "ETag", - "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" - }, - { - "key": "Date", - "value": "Tue, 30 Jul 2024 09:55:51 GMT" - }, - { - "key": "Connection", - "value": "keep-alive" - }, - { - "key": "Keep-Alive", - "value": "timeout=5" - } - ], - "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" - } - ] - } - ] - } - ] -} \ No newline at end of file diff --git a/api-service/postman-collection/updated_v2_collection.json b/api-service/postman-collection/updated_v2_collection.json index 45dc3ac3..b673ccfb 100644 --- a/api-service/postman-collection/updated_v2_collection.json +++ b/api-service/postman-collection/updated_v2_collection.json @@ -1,10 +1,9 @@ { "info": { - "_postman_id": "99bdb8d0-6d3a-4ba2-9044-ec5263e54c95", + "_postman_id": "51471244-e9b6-453e-964d-dccbdac80e72", "name": "V2 docs", "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json", - "_exporter_id": "33916975", - "_collection_link": "https://speeding-star-177775.postman.co/workspace/sanketika-obsrv~2ce96556-12e2-48bd-8e42-9c1dba428cc8/collection/33916975-99bdb8d0-6d3a-4ba2-9044-ec5263e54c95?action=share&source=collection_link&creator=33916975" + "_exporter_id": "37164806" }, "item": [ { @@ -2255,6 +2254,290 @@ ] } ] + }, + { + "name": "Connector api's", + "item": [ + { + "name": "Connector list", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "response": [ + { + "name": "Success: Filtered based on category", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6dc\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "1908" + }, + { + "key": "ETag", + "value": "W/\"774-58gfxDsxY66h3KZnTg62QbkxcPY\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:37:42 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:07:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"abc\",\n \"resmsgid\": \"632d3342-fd8a-47f7-afbb-96402a00b92f\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 3\n }\n}" + }, + { + "name": "Failure: Invalid request body, filter option array should not be empty", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "399" + }, + { + "key": "ETag", + "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:43:14 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + }, + { + "name": "Success: Connectors list with all filter options", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4107" + }, + { + "key": "ETag", + "value": "W/\"100b-SEJt8xn1TwmZz8DKpFAwkYx6E8s\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:47:51 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:17:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"fffa3ee0-da12-4bea-9b72-365571a62a4e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + }, + { + "name": "Success: Connectors list without filters", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "4107" + }, + { + "key": "ETag", + "value": "W/\"100b-N8mD4H5+SlIOmnNcq6Fnlq2pwXs\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:50:11 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:20:11+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c527a46-7bfe-4c74-a290-4dcf1a3c5f10\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + }, + { + "name": "Success: Filtered based on status", + "originalRequest": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": "http://localhost:3000/v2/connectors/list" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "250" + }, + { + "key": "ETag", + "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" + }, + { + "key": "Date", + "value": "Tue, 30 Jul 2024 09:55:51 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + } + ] + } + ] } ] } \ No newline at end of file From cbfcc50da24474ef07e05d8a9cc4826ccae9b867 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 30 Jul 2024 22:29:29 +0530 Subject: [PATCH 092/235] #OBS-I116: feat: Dataset Import and export api integration fixes --- .../RequestValidationSchemaV1.json | 3 +-- .../RequestValidationSchemaV2.json | 2 +- api-service/src/services/DatasetService.ts | 17 +++++++++++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json index 21a4b795..41acd1f2 100644 --- a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV1.json @@ -223,8 +223,7 @@ } } }, - "required": ["metadata", "data_schema"], - "additionalProperties": false + "required": ["metadata", "data_schema"] } }, "required": ["data"] diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json index 8efca393..45820502 100644 --- a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json @@ -409,7 +409,7 @@ "minLength": 1 }, "connector_config": { - "oneOf": [{ "type": "string" }, { "type": "object" }] + "type": "object" }, "operations_config": { "type": "object" diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index bb1d3efc..1f56c296 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -108,18 +108,20 @@ class DatasetService { return await this.getDraftDataset(datasetId); } - migrateDatasetV1= async (dataset_id: string, dataset: Record): Promise => { + migrateDatasetV1 = async (dataset_id: string, dataset: Record): Promise => { + const status = _.get(dataset, "status") let draftDataset: Record = { api_version: "v2", version_key: Date.now().toString() } const dataset_config: any = _.get(dataset, "dataset_config"); draftDataset["dataset_config"] = { - indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, - keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, - cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master") }, + keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, + cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } } - const transformations = await this.getDraftTransformations(dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); + const transformationFields = ["field_key", "transformation_function", "mode", "metadata"] + const transformations = _.includes([DatasetStatus.Live], status) ? await this.getTransformations(dataset_id, transformationFields) : await this.getDraftTransformations(dataset_id, transformationFields); draftDataset["transformations_config"] = _.map(transformations, (config) => { return { field_key: _.get(config, ["field_key"]), @@ -131,7 +133,8 @@ class DatasetService { mode: _.get(config, ["mode"]) } }) - const connectors = await this.getDraftConnectors(dataset_id, ["id", "connector_type", "connector_config"]); + const connectorsFields = ["id", "connector_type", "connector_config"] + const connectors = _.includes([DatasetStatus.Live], status) ? await this.getConnectorsV1(dataset_id, connectorsFields) : await this.getDraftConnectors(dataset_id, connectorsFields); draftDataset["connectors_config"] = _.map(connectors, (config) => { return { id: _.get(config, ["id"]), @@ -140,6 +143,7 @@ class DatasetService { version: "v1" } }) + draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) return draftDataset; } @@ -186,6 +190,7 @@ class DatasetService { } }) draftDataset["api_version"] = "v2" + draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) } else { const connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); draftDataset["connectors_config"] = connectors From 3aa4029babe1561d5fb024851f5a5b473915f9d9 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 31 Jul 2024 11:08:50 +0530 Subject: [PATCH 093/235] #OBS-I138: removed comment --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 65a175fe..b38ec5a5 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -184,7 +184,6 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) const updateMasterDataConfig = async (draftDataset: Record) => { if (draftDataset.type === 'master') { - // _.set(draftDataset,"datasetConfig.cache_config",datasetDefaultConfig) let dataset_config = _.get(draftDataset, "dataset_config") const datasetCacheConfig = _.get(defaultDatasetConfig, "dataset_config.cache_config") draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } From 9141bbccb08d74c405bb44e0b74e104c57e633bb Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 31 Jul 2024 11:32:38 +0530 Subject: [PATCH 094/235] #OBS-I138: throwing error if dataset is undefined --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 2ecc875d..28909ff3 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -35,18 +35,16 @@ const datasetRead = async (req: Request, res: Response) => { const { fields, mode } = req.query; const attributes = !fields ? defaultFields : _.split(fields, ","); const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes) + if (!dataset) { + throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); + } if (dataset.connectors_config) { dataset.connectors_config = dataset.connectors_config.map((connector: any) => ({ ...connector, connector_config: JSON.parse(cipherService.decrypt(connector.connector_config)) })); } - - if (!dataset) { - throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); - } else { - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); - } + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { From a700cb0b47c98a8d68370459f81b9780d9c1ef0b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 31 Jul 2024 12:31:45 +0530 Subject: [PATCH 095/235] #OBS-I142: added test cases for connector list --- .../ConnectorsList/ConnectorsList.spec.ts | 132 ++++++++ .../Connectors/ConnectorsList/Fixtures.ts | 298 ++++++++++++++++++ 2 files changed, 430 insertions(+) create mode 100644 api-service/src/tests/Connectors/ConnectorsList/ConnectorsList.spec.ts create mode 100644 api-service/src/tests/Connectors/ConnectorsList/Fixtures.ts diff --git a/api-service/src/tests/Connectors/ConnectorsList/ConnectorsList.spec.ts b/api-service/src/tests/Connectors/ConnectorsList/ConnectorsList.spec.ts new file mode 100644 index 00000000..832f4ff6 --- /dev/null +++ b/api-service/src/tests/Connectors/ConnectorsList/ConnectorsList.spec.ts @@ -0,0 +1,132 @@ +import app from "../../../app"; +import chai, { expect } from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import { TestInputsForConnectorsList } from "./Fixtures"; +import { ConnectorRegistry } from "../../../models/ConnectorRegistry"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const apiId = "api.connectors.list" +const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + + +describe("Connector List Api", () => { + afterEach(() => { + chai.spy.restore(); + }); + + it("Connectors list Success: With all the filters provided", (done) => { + chai.spy.on(ConnectorRegistry, "findAll", () => { + return Promise.resolve([TestInputsForConnectorsList.VALID_CONNECTORS_LIST]) + }) + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.REQUEST_WITH_BOTH_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([TestInputsForConnectorsList.VALID_CONNECTORS_LIST]) + result.should.be.eq(expectedResult) + done(); + }); + }) + + it("Connectors list Success: Without filters", (done) => { + chai.spy.on(ConnectorRegistry, "findAll", () => { + return Promise.resolve([TestInputsForConnectorsList.VALID_CONNECTORS_LIST]) + }) + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.REQUEST_WITHOUT_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([TestInputsForConnectorsList.VALID_CONNECTORS_LIST]) + result.should.be.eq(expectedResult) + done(); + }); + }) + + it("Connectors list Success: Filtered based on category", (done) => { + chai.spy.on(ConnectorRegistry, "findAll", () => { + return Promise.resolve([TestInputsForConnectorsList.VALID_CONNECTORS_LIST_CATEGORY]) + }) + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.REQUEST_WITH_CATEGORY_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([TestInputsForConnectorsList.VALID_CONNECTORS_LIST_CATEGORY]) + result.should.be.eq(expectedResult) + done(); + }); + }) + + it("Connectors list Success: Filtered based on status", (done) => { + chai.spy.on(ConnectorRegistry, "findAll", () => { + return Promise.resolve([TestInputsForConnectorsList.VALID_CONNECTORS_LIST_STATUS]) + }) + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.REQUEST_WITH_STATUS_FILTERS) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.count.should.be.eq(1) + res.body.params.msgid.should.be.eq(msgid) + const result = JSON.stringify(res.body.result.data) + const expectedResult = JSON.stringify([TestInputsForConnectorsList.VALID_CONNECTORS_LIST_STATUS]) + result.should.be.eq(expectedResult) + done(); + }); + }) + + it("Connectors list failure: Invalid request payload provided", (done) => { + chai + .request(app) + .post("/v2/connectors/list") + .send(TestInputsForConnectorsList.INVALID_REQUEST) + .end((err, res) => { + res.should.have.status(httpStatus.BAD_REQUEST); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.code.should.be.eq("CONNECTORS_LIST_INPUT_INVALID") + expect(res.body.error.message).to.match(/^(.+) must NOT have fewer than 1 items$/) + done(); + }); + }); + + +}) \ No newline at end of file diff --git a/api-service/src/tests/Connectors/ConnectorsList/Fixtures.ts b/api-service/src/tests/Connectors/ConnectorsList/Fixtures.ts new file mode 100644 index 00000000..af996eca --- /dev/null +++ b/api-service/src/tests/Connectors/ConnectorsList/Fixtures.ts @@ -0,0 +1,298 @@ +export const TestInputsForConnectorsList = { + INVALID_REQUEST: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { category: [] } + } + }, + REQUEST_WITHOUT_FILTERS: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": {} + }, + REQUEST_WITH_STATUS_FILTERS: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { status: ["Live"] } + } + }, + REQUEST_WITH_CATEGORY_FILTERS: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { + "category": [ + "Database" + ] + } + } + }, + REQUEST_WITH_BOTH_FILTERS: { + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "filters": { category: ["Database"], status: ["Live"] } + } + }, + + VALID_CONNECTORS_LIST:{ + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-07-30T15:17:51+05:30", + "params": { + "status": "SUCCESS", + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d", + "resmsgid": "fffa3ee0-da12-4bea-9b72-365571a62a4e" + }, + "responseCode": "OK", + "result": { + "data": [ + { + "id": "postgres-connector-1.0.0", + "connector_id": "postgres-connector", + "name": "PostgreSQL", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform", + "technology": "scala", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.732Z", + "updated_date": "2024-06-25T04:38:28.732Z" + }, + { + "id": "mysql-connector-1.0.0", + "connector_id": "mysql-connector", + "name": "MySQL", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform", + "technology": "scala", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.834Z", + "updated_date": "2024-06-25T04:38:28.834Z" + }, + { + "id": "oracle-connector-1.0.0", + "connector_id": "oracle-connector", + "name": "Oracle", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform", + "technology": "scala", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.840Z", + "updated_date": "2024-06-25T04:38:28.840Z" + }, + { + "id": "mssql-connector-1.0.0", + "connector_id": "mssql-connector", + "name": "MS SQL", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform", + "technology": "scala", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.847Z", + "updated_date": "2024-06-25T04:38:28.847Z" + }, + { + "id": "aws-s3-connector-0.1.0", + "connector_id": "aws-s3-connector", + "name": "AWS S3", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.237Z", + "updated_date": "2024-06-25T04:39:21.237Z" + }, + { + "id": "azure-blob-connector-0.1.0", + "connector_id": "azure-blob-connector", + "name": "Azure Blob Store", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.302Z", + "updated_date": "2024-06-25T04:39:21.302Z" + }, + { + "id": "gcs-connector-0.1.0", + "connector_id": "gcs-connector", + "name": "Google Cloud Storage", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The GCS Connector is used to move data from any Google Bucket to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.364Z", + "updated_date": "2024-06-25T04:39:21.364Z" + } + ], + "count": 7 + } + }, + VALID_CONNECTORS_LIST_CATEGORY:{ + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-07-30T15:07:42+05:30", + "params": { + "status": "SUCCESS", + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d", + "resmsgid": "632d3342-fd8a-47f7-afbb-96402a00b92f" + }, + "responseCode": "OK", + "result": { + "data": [ + { + "id": "aws-s3-connector-0.1.0", + "connector_id": "aws-s3-connector", + "name": "AWS S3", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.237Z", + "updated_date": "2024-06-25T04:39:21.237Z" + }, + { + "id": "azure-blob-connector-0.1.0", + "connector_id": "azure-blob-connector", + "name": "Azure Blob Store", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.302Z", + "updated_date": "2024-06-25T04:39:21.302Z" + }, + { + "id": "gcs-connector-0.1.0", + "connector_id": "gcs-connector", + "name": "Google Cloud Storage", + "type": "source", + "category": "File", + "version": "0.1.0", + "description": "The GCS Connector is used to move data from any Google Bucket to the Obsrv platform", + "technology": "python", + "runtime": "spark", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png", + "status": "Live", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:39:21.364Z", + "updated_date": "2024-06-25T04:39:21.364Z" + } + ], + "count": 3 + } + }, + VALID_CONNECTORS_LIST_STATUS:{ + "id": "api.connectors.list", + "ver": "v2", + "ts": "2024-07-30T15:25:51+05:30", + "params": { + "status": "SUCCESS", + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d", + "resmsgid": "f506e725-eed4-41df-86dc-2477d5c4d19a" + }, + "responseCode": "OK", + "result": { + "data": [], + "count": 0 + } + } +} + From f0e2e2b0dba6c0bfd9ca40a54fec7cfc6beb1e67 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 31 Jul 2024 18:17:05 +0530 Subject: [PATCH 096/235] #OBS-I138: merging dataset defaults to dataset draft record before saving --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index b38ec5a5..8fa81e65 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -95,10 +95,11 @@ const readyForPublish = async (dataset: Record) => { let draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) - if (draftDataset?.type === 'master') { - defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); - } - _.merge(draftDataset, defaultConfigs) + _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue) => { + if (_.isBoolean(objValue) && _.isBoolean(srcValue)) { + return objValue; + } + }); const datasetValid: Record = schemaValidation(draftDataset, ReadyToPublishSchema) if (!datasetValid.isValid) { throw { From 015313c03c6f633f70c80efb1c4bdd6382790ef7 Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 31 Jul 2024 18:19:07 +0530 Subject: [PATCH 097/235] #OBS-I138: adding merged event to dataset while migrating live or draft dataset --- api-service/src/services/DatasetService.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index eca4bb67..ffe87026 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -124,6 +124,7 @@ class DatasetService { version: "v1" } }) + draftDataset["sample_data"] = dataset_config?.mergedEvent const transaction = await sequelize.transaction(); try { @@ -182,6 +183,7 @@ class DatasetService { } }) draftDataset["api_version"] = "v2" + draftDataset["sample_data"] = dataset_config?.mergedEvent } else { const connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); draftDataset["connectors_config"] = connectors From 64035b5752224123ef2bec440b505a2b3c241c6d Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 31 Jul 2024 19:01:46 +0530 Subject: [PATCH 098/235] #OBS-I142: added live_date field to defaultFields --- api-service/src/controllers/ConnectorsList/ConnectorsList.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts index a031d370..1dadd5ab 100644 --- a/api-service/src/controllers/ConnectorsList/ConnectorsList.ts +++ b/api-service/src/controllers/ConnectorsList/ConnectorsList.ts @@ -7,7 +7,7 @@ import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; import { connectorService } from "../../services/ConnectorService"; -const defaultFields = ["id", "connector_id", "name", "type", "category", "version", "description", "technology", "runtime", "licence", "owner", "iconurl", "status", "created_by", "updated_by", "created_date", "updated_date"]; +const defaultFields = ["id", "connector_id", "name", "type", "category", "version", "description", "technology", "runtime", "licence", "owner", "iconurl", "status", "created_by", "updated_by", "created_date", "updated_date", "live_date"]; const validateRequest = (req: Request) => { const isRequestValid: Record = schemaValidation(req.body, ConnectorListSchema) From fe1620adb6cf02596d25fe364e5577cd80fa82b0 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 31 Jul 2024 19:05:44 +0530 Subject: [PATCH 099/235] #OBS-I145: Connector Read API --- .../ConnectorsRead/ConnectorsRead.ts | 42 +++++++++++++++++++ api-service/src/routes/Router.ts | 2 + api-service/src/services/ConnectorService.ts | 8 ++++ 3 files changed, 52 insertions(+) create mode 100644 api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts diff --git a/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts b/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts new file mode 100644 index 00000000..49e62640 --- /dev/null +++ b/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts @@ -0,0 +1,42 @@ +import { Request, Response } from "express"; +import { obsrvError } from "../../types/ObsrvError"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import httpStatus from "http-status"; +import { connectorService } from "../../services/ConnectorService"; + +const defaultFields = ["id", "connector_id", "name", "type", "category", "version", "description", "licence", "owner", "iconurl", "status", "ui_spec", "created_by", "updated_by", "created_date", "updated_date", "live_date"]; + +const validateRequest = (req: Request) => { + + const { id } = req.params; + const mode = req.query.mode; + + if (mode && mode !== "edit") { + throw obsrvError(id, "DATASET_INVALID_MODE_VALUE", `The specified mode [${mode}] in the query param is not valid.`, "BAD_REQUEST", 400); + } + +} + +const connectorsRead = async (req: Request, res: Response) => { + validateRequest(req) + const { id } = req.params; + const { mode } = req.query; + const connector = (mode == "edit") ? await readDraftConnector(id, defaultFields) : await readConnector(id, defaultFields) + if (!connector) { + throw obsrvError(id, "CONNECTOR_NOT_FOUND", `Connector with the given id: ${id} not found`, "NOT_FOUND", 404); + } else { + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: connector }); + } +} + +const readDraftConnector = async (Id: string, defaultFields: string[]): Promise => { + const connector = await connectorService.getDraftConnector(Id, "Draft", defaultFields); + return connector; +} + +const readConnector = async (Id: string, defaultFields: string[]): Promise => { + const connector = await connectorService.getConnector(Id, defaultFields); + return connector; +} + +export default connectorsRead; \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 4ba84153..03721be9 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -25,6 +25,7 @@ import datasetReset from "../controllers/DatasetReset/DatasetReset"; import DatasetExport from "../controllers/DatasetExport/DatasetExport"; import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; +import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; export const router = express.Router(); @@ -50,6 +51,7 @@ router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.datasch router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), DatasetCopy); router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), ConnectorsRead); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file diff --git a/api-service/src/services/ConnectorService.ts b/api-service/src/services/ConnectorService.ts index 3b7a5340..a10d3f53 100644 --- a/api-service/src/services/ConnectorService.ts +++ b/api-service/src/services/ConnectorService.ts @@ -6,6 +6,14 @@ class ConnectorService { return ConnectorRegistry.findAll({ where, attributes, raw: true }); } + getConnector = async (Id: string, attributes?: string[]): Promise => { + return ConnectorRegistry.findOne({ where: { id: Id }, attributes }); + } + + getDraftConnector = async (Id: string, Status: string, attributes?: string[]): Promise => { + return ConnectorRegistry.findOne({ where: { id: Id, status: Status }, attributes }); + } + } export const connectorService = new ConnectorService(); \ No newline at end of file From 706fc1e0d44bb4b97eb76eb607bb11ba50970f3f Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 31 Jul 2024 21:57:21 +0530 Subject: [PATCH 100/235] #OBS-I145: updated postman collection with connector read api --- .../updated_v2_collection.json | 245 +++++++++++++++--- 1 file changed, 211 insertions(+), 34 deletions(-) diff --git a/api-service/postman-collection/updated_v2_collection.json b/api-service/postman-collection/updated_v2_collection.json index b673ccfb..0552c08d 100644 --- a/api-service/postman-collection/updated_v2_collection.json +++ b/api-service/postman-collection/updated_v2_collection.json @@ -2276,13 +2276,13 @@ }, "response": [ { - "name": "Success: Filtered based on category", + "name": "Failure: Invalid request body, filter option array should not be empty", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6dc\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -2291,8 +2291,8 @@ }, "url": "http://localhost:3000/v2/connectors/list" }, - "status": "OK", - "code": 200, + "status": "Bad Request", + "code": 400, "_postman_previewlanguage": "json", "header": [ { @@ -2305,15 +2305,15 @@ }, { "key": "Content-Length", - "value": "1908" + "value": "399" }, { "key": "ETag", - "value": "W/\"774-58gfxDsxY66h3KZnTg62QbkxcPY\"" + "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:37:42 GMT" + "value": "Tue, 30 Jul 2024 09:43:14 GMT" }, { "key": "Connection", @@ -2325,16 +2325,16 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:07:42+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"abc\",\n \"resmsgid\": \"632d3342-fd8a-47f7-afbb-96402a00b92f\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 3\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" }, { - "name": "Failure: Invalid request body, filter option array should not be empty", + "name": "Success: Filtered based on status", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\"\n ],\n \"status\": [\n //\"Live\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -2343,8 +2343,8 @@ }, "url": "http://localhost:3000/v2/connectors/list" }, - "status": "Bad Request", - "code": 400, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2357,15 +2357,15 @@ }, { "key": "Content-Length", - "value": "399" + "value": "250" }, { "key": "ETag", - "value": "W/\"18f-Hsau3RTrCuWgbSoS3cqIWuUq45k\"" + "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:43:14 GMT" + "value": "Tue, 30 Jul 2024 09:55:51 GMT" }, { "key": "Connection", @@ -2377,16 +2377,16 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:13:14+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"02fadde0-8c59-4420-8ab3-56474b01670b\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTORS_LIST_INPUT_INVALID\",\n \"message\": \"#properties/request/properties/filters/properties/status/minItems must NOT have fewer than 1 items\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" }, { - "name": "Success: Connectors list with all filter options", + "name": "Success: Filtered based on category", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"Database\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -2409,15 +2409,15 @@ }, { "key": "Content-Length", - "value": "4107" + "value": "2571" }, { "key": "ETag", - "value": "W/\"100b-SEJt8xn1TwmZz8DKpFAwkYx6E8s\"" + "value": "W/\"a0b-YhLfH2KW3BX83ncggqexRrMMI6E\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:47:51 GMT" + "value": "Wed, 31 Jul 2024 13:25:03 GMT" }, { "key": "Connection", @@ -2429,16 +2429,16 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:17:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"fffa3ee0-da12-4bea-9b72-365571a62a4e\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:55:03+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n ],\n \"count\": 4\n }\n}" }, { - "name": "Success: Connectors list without filters", + "name": "Success: Connectors list with all filter options", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"category\":[\n \"File\",\"Database\"\n ],\n \"status\": [\n \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n ]\n }\n }\n}", "options": { "raw": { "language": "json" @@ -2461,15 +2461,15 @@ }, { "key": "Content-Length", - "value": "4107" + "value": "4380" }, { "key": "ETag", - "value": "W/\"100b-N8mD4H5+SlIOmnNcq6Fnlq2pwXs\"" + "value": "W/\"111c-nqfT0Ww3TEj5mK7ut9ZCkyIXz2I\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:50:11 GMT" + "value": "Wed, 31 Jul 2024 13:26:32 GMT" }, { "key": "Connection", @@ -2481,16 +2481,16 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:20:11+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"2c527a46-7bfe-4c74-a290-4dcf1a3c5f10\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:56:32+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"11a2f537-bd98-405b-97e5-0f0d5b86b2c3\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" }, { - "name": "Success: Filtered based on status", + "name": "Success: Connectors list without filters", "originalRequest": { "method": "POST", "header": [], "body": { "mode": "raw", - "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n \"filters\": {\n \"status\": [\n \"Draft\"\n ]\n }\n }\n}", + "raw": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-04-10T16:10:50+05:30\",\n \"params\": {\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\"\n },\n \"request\": {\n // \"filters\": {\n // \"category\":[\n // \"File\",\"Database\"\n // ],\n // \"status\": [\n // \"Live\",\"Draft\",\"InValidation\", \"Retired\"\n // ]\n // }\n }\n}", "options": { "raw": { "language": "json" @@ -2513,15 +2513,15 @@ }, { "key": "Content-Length", - "value": "250" + "value": "4380" }, { "key": "ETag", - "value": "W/\"fa-+eWKIfUxsWBGuJy23qSucgLXke4\"" + "value": "W/\"111c-GYs9s/7ILhe56TljQaYO8fXzKGU\"" }, { "key": "Date", - "value": "Tue, 30 Jul 2024 09:55:51 GMT" + "value": "Wed, 31 Jul 2024 13:27:37 GMT" }, { "key": "Connection", @@ -2533,7 +2533,184 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-30T15:25:51+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"f506e725-eed4-41df-86dc-2477d5c4d19a\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [],\n \"count\": 0\n }\n}" + "body": "{\n \"id\": \"api.connectors.list\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:57:37+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"msgid\": \"4a7f14c3-d61e-4d4f-be78-181834eeff6d\",\n \"resmsgid\": \"c2467e01-0a2d-401c-aa3d-dd16b804f723\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"data\": [\n {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n },\n {\n \"id\": \"mysql-connector-1.0.0\",\n \"connector_id\": \"mysql-connector\",\n \"name\": \"MySQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MySQL Connector is used to move data from any MySQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.834Z\",\n \"updated_date\": \"2024-06-25T04:38:28.834Z\",\n \"live_date\": \"2024-06-25T04:38:28.834Z\"\n },\n {\n \"id\": \"oracle-connector-1.0.0\",\n \"connector_id\": \"oracle-connector\",\n \"name\": \"Oracle\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The Oracle Connector is used to move data from any Oracle Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.840Z\",\n \"updated_date\": \"2024-06-25T04:38:28.840Z\",\n \"live_date\": \"2024-06-25T04:38:28.840Z\"\n },\n {\n \"id\": \"mssql-connector-1.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"technology\": \"scala\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n },\n {\n \"id\": \"aws-s3-connector-0.1.0\",\n \"connector_id\": \"aws-s3-connector\",\n \"name\": \"AWS S3\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The AWS S3 Connector is used to move data from any S3 Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.237Z\",\n \"updated_date\": \"2024-06-25T04:39:21.237Z\",\n \"live_date\": \"2024-06-25T04:39:21.237Z\"\n },\n {\n \"id\": \"azure-blob-connector-0.1.0\",\n \"connector_id\": \"azure-blob-connector\",\n \"name\": \"Azure Blob Store\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The Azure Blob Store Connector is used to move data from any Azure Blob Container to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.302Z\",\n \"updated_date\": \"2024-06-25T04:39:21.302Z\",\n \"live_date\": \"2024-06-25T04:39:21.302Z\"\n },\n {\n \"id\": \"gcs-connector-0.1.0\",\n \"connector_id\": \"gcs-connector\",\n \"name\": \"Google Cloud Storage\",\n \"type\": \"source\",\n \"category\": \"File\",\n \"version\": \"0.1.0\",\n \"description\": \"The GCS Connector is used to move data from any Google Bucket to the Obsrv platform\",\n \"technology\": \"python\",\n \"runtime\": \"spark\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png\",\n \"status\": \"Live\",\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:39:21.364Z\",\n \"updated_date\": \"2024-06-25T04:39:21.364Z\",\n \"live_date\": \"2024-06-25T04:39:21.364Z\"\n }\n ],\n \"count\": 7\n }\n}" + } + ] + }, + { + "name": "Connector Read", + "request": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + }, + "response": [ + { + "name": "Success: Read live Connector", + "originalRequest": { + "method": "GET", + "header": [], + "url": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0" + }, + "status": "OK", + "code": 200, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "3304" + }, + { + "key": "ETag", + "value": "W/\"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 12:47:54 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" + }, + { + "name": "Failure: Connector not found", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0?mode=edit", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-connector-1.0.0" + ], + "query": [ + { + "key": "mode", + "value": "edit" + } + ] + } + }, + "status": "Not Found", + "code": 404, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "305" + }, + { + "key": "ETag", + "value": "W/\"131-34hIJjn0HC4sZfrhBno7pcto+x4\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 12:49:17 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:19:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"c3333acf-761e-4601-8254-850fd8641494\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector with the given id: postgres-connector-1.0.0 not found\"\n }\n}" + }, + { + "name": "Failure: Invalid mode value", + "originalRequest": { + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0?mode=23", + "protocol": "http", + "host": [ + "localhost" + ], + "port": "3000", + "path": [ + "v2", + "connectors", + "read", + "postgres-connector-1.0.0" + ], + "query": [ + { + "key": "mode", + "value": "23" + } + ] + } + }, + "status": "Bad Request", + "code": 400, + "_postman_previewlanguage": "json", + "header": [ + { + "key": "X-Powered-By", + "value": "Express" + }, + { + "key": "Content-Type", + "value": "application/json; charset=utf-8" + }, + { + "key": "Content-Length", + "value": "307" + }, + { + "key": "ETag", + "value": "W/\"133-NqsLTfEXPavK/auClQsE69MITWw\"" + }, + { + "key": "Date", + "value": "Wed, 31 Jul 2024 13:17:54 GMT" + }, + { + "key": "Connection", + "value": "keep-alive" + }, + { + "key": "Keep-Alive", + "value": "timeout=5" + } + ], + "cookie": [], + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:47:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"5ce8b69d-cda2-4035-8b82-d8b64cf3c574\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_MODE_VALUE\",\n \"message\": \"The specified mode [23] in the query param is not valid.\"\n }\n}" } ] } From afa746f3de31ee32c06161ae495d0a6c5bc50fe7 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 31 Jul 2024 21:58:01 +0530 Subject: [PATCH 101/235] #OBS-I145: updated swagger documentation --- .../swagger-doc/v2_updated_doc_openapi.yml | 458 +++++++++++++++--- 1 file changed, 400 insertions(+), 58 deletions(-) diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml index 48400bff..c95a15a7 100644 --- a/api-service/swagger-doc/v2_updated_doc_openapi.yml +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -4206,89 +4206,127 @@ paths: type: object examples: example-0: + summary: 'Success: Filtered based on status' + value: + id: api.connectors.list + ver: v2 + ts: '2024-07-30T15:25:51+05:30' + params: + status: SUCCESS + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: f506e725-eed4-41df-86dc-2477d5c4d19a + responseCode: OK + result: + data: [] + count: 0 + example-1: summary: 'Success: Filtered based on category' value: id: api.connectors.list ver: v2 - ts: '2024-07-30T15:07:42+05:30' + ts: '2024-07-31T18:55:03+05:30' params: status: SUCCESS - msgid: abc - resmsgid: 632d3342-fd8a-47f7-afbb-96402a00b92f + msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d + resmsgid: e3a0dbff-daad-4bdd-abd4-6bb5e1e30cab responseCode: OK result: data: - - id: aws-s3-connector-0.1.0 - connector_id: aws-s3-connector - name: AWS S3 + - id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL type: source - category: File - version: 0.1.0 + category: Database + version: 1.0.0 description: >- - The AWS S3 Connector is used to move data from any - S3 Bucket to the Obsrv platform - technology: python + The PostgreSQL Connector is used to move data from + any Postgres Table to the Obsrv platform + technology: scala runtime: spark licence: MIT owner: Sunbird iconurl: >- - https://upload.wikimedia.org/wikipedia/commons/b/bc/Amazon-S3-Logo.svg + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg status: Live created_by: SYSTEM updated_by: SYSTEM - created_date: '2024-06-25T04:39:21.237Z' - updated_date: '2024-06-25T04:39:21.237Z' - - id: azure-blob-connector-0.1.0 - connector_id: azure-blob-connector - name: Azure Blob Store + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + - id: mysql-connector-1.0.0 + connector_id: mysql-connector + name: MySQL type: source - category: File - version: 0.1.0 + category: Database + version: 1.0.0 description: >- - The Azure Blob Store Connector is used to move data - from any Azure Blob Container to the Obsrv platform - technology: python + The MySQL Connector is used to move data from any + MySQL Table to the Obsrv platform + technology: scala runtime: spark licence: MIT owner: Sunbird iconurl: >- - https://upload.wikimedia.org/wikipedia/commons/f/fa/Microsoft_Azure.svg + https://upload.wikimedia.org/wikipedia/en/6/62/MySQL.svg status: Live created_by: SYSTEM updated_by: SYSTEM - created_date: '2024-06-25T04:39:21.302Z' - updated_date: '2024-06-25T04:39:21.302Z' - - id: gcs-connector-0.1.0 - connector_id: gcs-connector - name: Google Cloud Storage + created_date: '2024-06-25T04:38:28.834Z' + updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' + - id: oracle-connector-1.0.0 + connector_id: oracle-connector + name: Oracle type: source - category: File - version: 0.1.0 + category: Database + version: 1.0.0 description: >- - The GCS Connector is used to move data from any - Google Bucket to the Obsrv platform - technology: python + The Oracle Connector is used to move data from any + Oracle Table to the Obsrv platform + technology: scala runtime: spark licence: MIT owner: Sunbird iconurl: >- - https://upload.wikimedia.org/wikipedia/commons/thumb/5/51/Google_Cloud_logo.svg/512px-Google_Cloud_logo.svg.png + https://upload.wikimedia.org/wikipedia/commons/5/50/Oracle_logo.svg status: Live created_by: SYSTEM updated_by: SYSTEM - created_date: '2024-06-25T04:39:21.364Z' - updated_date: '2024-06-25T04:39:21.364Z' - count: 3 - example-1: + created_date: '2024-06-25T04:38:28.840Z' + updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' + - id: mssql-connector-1.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 1.0.0 + description: >- + The MS SQL Connector is used to move data from any + MS SQL Table to the Obsrv platform + technology: scala + runtime: spark + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Live + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' + count: 4 + example-2: summary: 'Success: Connectors list with all filter options' value: id: api.connectors.list ver: v2 - ts: '2024-07-30T15:17:51+05:30' + ts: '2024-07-31T18:56:32+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: fffa3ee0-da12-4bea-9b72-365571a62a4e + resmsgid: 11a2f537-bd98-405b-97e5-0f0d5b86b2c3 responseCode: OK result: data: @@ -4312,6 +4350,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:38:28.732Z' updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' - id: mysql-connector-1.0.0 connector_id: mysql-connector name: MySQL @@ -4332,6 +4371,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:38:28.834Z' updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' - id: oracle-connector-1.0.0 connector_id: oracle-connector name: Oracle @@ -4352,6 +4392,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:38:28.840Z' updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' - id: mssql-connector-1.0.0 connector_id: mssql-connector name: MS SQL @@ -4372,6 +4413,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:38:28.847Z' updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' - id: aws-s3-connector-0.1.0 connector_id: aws-s3-connector name: AWS S3 @@ -4392,6 +4434,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:39:21.237Z' updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' - id: azure-blob-connector-0.1.0 connector_id: azure-blob-connector name: Azure Blob Store @@ -4412,6 +4455,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:39:21.302Z' updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' - id: gcs-connector-0.1.0 connector_id: gcs-connector name: Google Cloud Storage @@ -4432,17 +4476,18 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:39:21.364Z' updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' count: 7 - example-2: + example-3: summary: 'Success: Connectors list without filters' value: id: api.connectors.list ver: v2 - ts: '2024-07-30T15:20:11+05:30' + ts: '2024-07-31T18:57:37+05:30' params: status: SUCCESS msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: 2c527a46-7bfe-4c74-a290-4dcf1a3c5f10 + resmsgid: c2467e01-0a2d-401c-aa3d-dd16b804f723 responseCode: OK result: data: @@ -4466,6 +4511,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:38:28.732Z' updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' - id: mysql-connector-1.0.0 connector_id: mysql-connector name: MySQL @@ -4486,6 +4532,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:38:28.834Z' updated_date: '2024-06-25T04:38:28.834Z' + live_date: '2024-06-25T04:38:28.834Z' - id: oracle-connector-1.0.0 connector_id: oracle-connector name: Oracle @@ -4506,6 +4553,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:38:28.840Z' updated_date: '2024-06-25T04:38:28.840Z' + live_date: '2024-06-25T04:38:28.840Z' - id: mssql-connector-1.0.0 connector_id: mssql-connector name: MS SQL @@ -4526,6 +4574,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:38:28.847Z' updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' - id: aws-s3-connector-0.1.0 connector_id: aws-s3-connector name: AWS S3 @@ -4546,6 +4595,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:39:21.237Z' updated_date: '2024-06-25T04:39:21.237Z' + live_date: '2024-06-25T04:39:21.237Z' - id: azure-blob-connector-0.1.0 connector_id: azure-blob-connector name: Azure Blob Store @@ -4566,6 +4616,7 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:39:21.302Z' updated_date: '2024-06-25T04:39:21.302Z' + live_date: '2024-06-25T04:39:21.302Z' - id: gcs-connector-0.1.0 connector_id: gcs-connector name: Google Cloud Storage @@ -4586,21 +4637,8 @@ paths: updated_by: SYSTEM created_date: '2024-06-25T04:39:21.364Z' updated_date: '2024-06-25T04:39:21.364Z' + live_date: '2024-06-25T04:39:21.364Z' count: 7 - example-3: - summary: 'Success: Filtered based on status' - value: - id: api.connectors.list - ver: v2 - ts: '2024-07-30T15:25:51+05:30' - params: - status: SUCCESS - msgid: 4a7f14c3-d61e-4d4f-be78-181834eeff6d - resmsgid: f506e725-eed4-41df-86dc-2477d5c4d19a - responseCode: OK - result: - data: [] - count: 0 '400': description: Bad Request content: @@ -4621,4 +4659,308 @@ paths: code: CONNECTORS_LIST_INPUT_INVALID message: >- #properties/request/properties/filters/properties/status/minItems - must NOT have fewer than 1 items \ No newline at end of file + must NOT have fewer than 1 items + /v2/connectors/read/postgres-connector-1.0.0: + get: + tags: + - Connector api's + summary: Connector Read + responses: + '200': + description: OK + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '3304' + ETag: + schema: + type: string + example: W/"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28" + Date: + schema: + type: string + example: Wed, 31 Jul 2024 12:47:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.connectors.read + ver: v2 + ts: '2024-07-31T18:17:54+05:30' + params: + status: SUCCESS + resmsgid: 7587f564-c2d7-49a8-9e56-dc56f6808ced + responseCode: OK + result: + id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from any + Postgres Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + ui_spec: + schema: + type: object + properties: + connector_config: + title: Connector Config + type: object + encrypt: true + properties: + databaseType: + type: string + title: Database Type + enum: + - PostgreSQL + - MySQL + fieldDescription: + - type: string + description: '' + dependencies: + databaseType: + oneOf: + - properties: + databaseType: + enum: + - PostgreSQL + - MySQL + connection_info: + title: Connection Information + type: object + properties: + host: + type: string + title: Database Host + pattern: >- + /^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(?:/[^\s]*)?|localhost(?:/[^\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/ + fieldDescription: + - type: string + description: '' + port: + type: number + title: Database Port + minimum: 1 + maximum: 65535 + fieldDescription: + - type: string + description: '' + name: + type: string + title: Database Name + pattern: ^[a-zA-Z0-9_]{1,64}$ + fieldDescription: + - type: string + description: '' + username: + type: string + title: Database Username + fieldDescription: + - type: string + description: '' + password: + type: string + title: Database Password + fieldDescription: + - type: string + description: '' + schemaInfo: + title: Schema Information + type: object + properties: + table: + title: Table Name + type: string + pattern: ^[a-zA-Z_][a-zA-Z0-9_]{0,62}$ + fieldDescription: + - type: string + description: '' + timestampColumn: + title: Timestamp Column + type: string + fieldDescription: + - type: string + description: '' + operations_config: + title: Operations Configuration + type: object + properties: + batch_size: + type: number + title: Batch Size + default: 100 + fieldDescription: + - type: string + description: '' + max_batches: + type: number + title: Maximum Batches + default: 10 + fieldDescription: + - type: string + description: '' + pollingInterval: + type: string + title: Polling Interval + enum: + - Once + - Periodic + fieldDescription: + - type: string + description: Select polling interval + dependencies: + pollingInterval: + oneOf: + - properties: + pollingInterval: + enum: + - Periodic + schedule: + type: string + title: Schedule + enum: + - Hourly + - Daily + - Weekly + - Monthly + fieldDescription: + - type: string + description: '' + required: + - schedule + properties: + connector_config: + connection_info: + password: + ui:widget: password + operations_config: + batch_size: + ui:readonly: true + max_batches: + ui:readonly: true + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + '400': + description: Bad Request + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '307' + ETag: + schema: + type: string + example: W/"133-NqsLTfEXPavK/auClQsE69MITWw" + Date: + schema: + type: string + example: Wed, 31 Jul 2024 13:17:54 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.connectors.read + ver: v2 + ts: '2024-07-31T18:47:54+05:30' + params: + status: FAILED + resmsgid: 5ce8b69d-cda2-4035-8b82-d8b64cf3c574 + responseCode: BAD_REQUEST + result: {} + error: + code: DATASET_INVALID_MODE_VALUE + message: The specified mode [23] in the query param is not valid. + '404': + description: Not Found + headers: + X-Powered-By: + schema: + type: string + example: Express + Content-Type: + schema: + type: string + example: application/json; charset=utf-8 + Content-Length: + schema: + type: integer + example: '305' + ETag: + schema: + type: string + example: W/"131-34hIJjn0HC4sZfrhBno7pcto+x4" + Date: + schema: + type: string + example: Wed, 31 Jul 2024 12:49:17 GMT + Connection: + schema: + type: string + example: keep-alive + Keep-Alive: + schema: + type: string + example: timeout=5 + content: + application/json: + schema: + type: object + example: + id: api.connectors.read + ver: v2 + ts: '2024-07-31T18:19:17+05:30' + params: + status: FAILED + resmsgid: c3333acf-761e-4601-8254-850fd8641494 + responseCode: NOT_FOUND + result: {} + error: + code: CONNECTOR_NOT_FOUND + message: >- + Connector with the given id: postgres-connector-1.0.0 not + found \ No newline at end of file From 0a58cbbe051fb9d1fd50ae6e4cb3f5e22e7bf3f0 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 1 Aug 2024 12:41:06 +0530 Subject: [PATCH 102/235] #OBS-I145: updated the connector read api --- .../ConnectorsRead/ConnectorsRead.ts | 27 +++---------------- api-service/src/services/ConnectorService.ts | 8 ++---- 2 files changed, 5 insertions(+), 30 deletions(-) diff --git a/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts b/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts index 49e62640..8daa059d 100644 --- a/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts +++ b/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts @@ -6,37 +6,16 @@ import { connectorService } from "../../services/ConnectorService"; const defaultFields = ["id", "connector_id", "name", "type", "category", "version", "description", "licence", "owner", "iconurl", "status", "ui_spec", "created_by", "updated_by", "created_date", "updated_date", "live_date"]; -const validateRequest = (req: Request) => { - - const { id } = req.params; - const mode = req.query.mode; - - if (mode && mode !== "edit") { - throw obsrvError(id, "DATASET_INVALID_MODE_VALUE", `The specified mode [${mode}] in the query param is not valid.`, "BAD_REQUEST", 400); - } - -} - const connectorsRead = async (req: Request, res: Response) => { - validateRequest(req) const { id } = req.params; const { mode } = req.query; - const connector = (mode == "edit") ? await readDraftConnector(id, defaultFields) : await readConnector(id, defaultFields) + + const connector = (mode?.toString()?.toLowerCase()) === "edit" ? await connectorService.getConnector({ id: id, status: "Draft" }, defaultFields) : await connectorService.getConnector({ id: id, status: "Live" }, defaultFields); if (!connector) { - throw obsrvError(id, "CONNECTOR_NOT_FOUND", `Connector with the given id: ${id} not found`, "NOT_FOUND", 404); + throw obsrvError(id, "CONNECTOR_NOT_FOUND", `Connector not found: ${id}`, "NOT_FOUND", 404); } else { ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: connector }); } } -const readDraftConnector = async (Id: string, defaultFields: string[]): Promise => { - const connector = await connectorService.getDraftConnector(Id, "Draft", defaultFields); - return connector; -} - -const readConnector = async (Id: string, defaultFields: string[]): Promise => { - const connector = await connectorService.getConnector(Id, defaultFields); - return connector; -} - export default connectorsRead; \ No newline at end of file diff --git a/api-service/src/services/ConnectorService.ts b/api-service/src/services/ConnectorService.ts index a10d3f53..b937180a 100644 --- a/api-service/src/services/ConnectorService.ts +++ b/api-service/src/services/ConnectorService.ts @@ -6,12 +6,8 @@ class ConnectorService { return ConnectorRegistry.findAll({ where, attributes, raw: true }); } - getConnector = async (Id: string, attributes?: string[]): Promise => { - return ConnectorRegistry.findOne({ where: { id: Id }, attributes }); - } - - getDraftConnector = async (Id: string, Status: string, attributes?: string[]): Promise => { - return ConnectorRegistry.findOne({ where: { id: Id, status: Status }, attributes }); + getConnector = async (where?: Record, attributes?: string[]): Promise => { + return ConnectorRegistry.findOne({ where, attributes, raw: true}); } } From 62679be3f235401550654dc502651671c605fb5e Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 1 Aug 2024 15:20:33 +0530 Subject: [PATCH 103/235] merge changes --- .../spark-connector-cron/templates/rbac.yaml | 48 ------------------- .../templates/serviceaccount.yaml | 11 ----- 2 files changed, 59 deletions(-) delete mode 100644 command-service/helm-charts/spark-connector-cron/templates/rbac.yaml delete mode 100644 command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml diff --git a/command-service/helm-charts/spark-connector-cron/templates/rbac.yaml b/command-service/helm-charts/spark-connector-cron/templates/rbac.yaml deleted file mode 100644 index 7e291fd6..00000000 --- a/command-service/helm-charts/spark-connector-cron/templates/rbac.yaml +++ /dev/null @@ -1,48 +0,0 @@ -{{- if .Values.rbac.enabled -}} ---- -{{- if .Values.rbac.useClusterRole }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: {{ include "common.names.fullname" . }} -rules: -{{- toYaml .Values.rbac.rules | nindent 2 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: {{ include "common.names.fullname" . }} -subjects: -- kind: ServiceAccount - name: {{ include "base.serviceaccountname" . }} - namespace: {{ include "base.namespace" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: {{ include "common.names.fullname" . }} -{{- else }} -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: {{ include "common.names.fullname" . }} - namespace: {{ include "base.namespace" . }} -rules: -{{- toYaml .Values.rbac.rules | nindent 2 }} ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: {{ include "common.names.fullname" . }} - namespace: {{ include "base.namespace" . }} -subjects: -- kind: ServiceAccount - name: {{ include "base.serviceaccountname" . }} - namespace: {{ include "base.namespace" . }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: {{ include "common.names.fullname" . }} -{{- end }} - -{{- end }} - diff --git a/command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml b/command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml deleted file mode 100644 index d26e8536..00000000 --- a/command-service/helm-charts/spark-connector-cron/templates/serviceaccount.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if .Values.serviceAccount.create }} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "base.serviceaccountname" . }} - namespace: {{ include "base.namespace" . }} - {{- if or .Values.serviceAccount.annotations .Values.commonAnnotations }} - {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceAccount.annotations .Values.commonAnnotations ) "context" . ) }} - annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} - {{- end }} -{{- end }} From f6d84c7d464346d9273efb0f9e80e6474d150394 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 1 Aug 2024 15:22:08 +0530 Subject: [PATCH 104/235] merge changes --- .../helm-charts/spark-connector-cron/templates/cronjob.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index f42637a4..07b70fb2 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -19,7 +19,7 @@ spec: {{- end }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 12 }} spec: - serviceAccountName: {{ include "base.serviceaccountname" . }} + serviceAccountName: {{ .Values.serviceAccount.name }} restartPolicy: {{ .Values.restartPolicy }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 12 }} From bb56fb415218cf24d067a4ca27038d0fa13e1c98 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 1 Aug 2024 15:24:57 +0530 Subject: [PATCH 105/235] Resolved merge changes --- command-service/src/command/connector_command.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index e5f67970..2dbed28d 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -132,8 +132,6 @@ def _perform_spark_install(self, connector_instance): "--set", "main_file={}".format(connector_source["main_program"]), "--set", - "serviceAccount.create=false", - "--set", "cronSchedule={}".format(connector_instance.operations_config["schedule"]) ] From d98a79db673bb09b7feb2d42bf5a98ab29f204e1 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 1 Aug 2024 15:25:59 +0530 Subject: [PATCH 106/235] Resolved merge changes --- .../helm-charts/spark-connector-cron/values.yaml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/command-service/helm-charts/spark-connector-cron/values.yaml b/command-service/helm-charts/spark-connector-cron/values.yaml index a781b613..1c4c2fce 100644 --- a/command-service/helm-charts/spark-connector-cron/values.yaml +++ b/command-service/helm-charts/spark-connector-cron/values.yaml @@ -92,20 +92,7 @@ configmap: mountPath: /config serviceAccount: - create: true - name: spark-cron-sa - annotations: {} - -rbac: - enabled: true - useClusterRole: true - rules: - - apiGroups: [""] - resources: ["pods"] - verbs: ["get", "list", "watch"] - - apiGroups: [""] - resources: ["pods/exec"] - verbs: ["create"] + name: spark-connectors-sa serviceMonitor: enabled: false From 63288845afe22bb6a52adff36a88ca0ebb49f7c2 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 1 Aug 2024 15:38:49 +0530 Subject: [PATCH 107/235] #OBS-I145: updated the connector read api --- .../src/controllers/ConnectorsRead/ConnectorsRead.ts | 11 ++++++----- api-service/src/services/ConnectorService.ts | 6 +++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts b/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts index 8daa059d..1e482e2c 100644 --- a/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts +++ b/api-service/src/controllers/ConnectorsRead/ConnectorsRead.ts @@ -3,19 +3,20 @@ import { obsrvError } from "../../types/ObsrvError"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import httpStatus from "http-status"; import { connectorService } from "../../services/ConnectorService"; +import _ from "lodash"; const defaultFields = ["id", "connector_id", "name", "type", "category", "version", "description", "licence", "owner", "iconurl", "status", "ui_spec", "created_by", "updated_by", "created_date", "updated_date", "live_date"]; const connectorsRead = async (req: Request, res: Response) => { const { id } = req.params; const { mode } = req.query; - - const connector = (mode?.toString()?.toLowerCase()) === "edit" ? await connectorService.getConnector({ id: id, status: "Draft" }, defaultFields) : await connectorService.getConnector({ id: id, status: "Live" }, defaultFields); + const isEditMode = _.toLower(_.toString(mode)) === "edit" + const status = isEditMode ? "Draft" : "Live" + const connector = await connectorService.getConnector({ id, status }, defaultFields) if (!connector) { - throw obsrvError(id, "CONNECTOR_NOT_FOUND", `Connector not found: ${id}`, "NOT_FOUND", 404); - } else { - ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: connector }); + throw obsrvError("", "CONNECTOR_NOT_FOUND", `Connector not found: ${id}`, "NOT_FOUND", 404); } + ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: connector }); } export default connectorsRead; \ No newline at end of file diff --git a/api-service/src/services/ConnectorService.ts b/api-service/src/services/ConnectorService.ts index b937180a..ddbb211b 100644 --- a/api-service/src/services/ConnectorService.ts +++ b/api-service/src/services/ConnectorService.ts @@ -2,12 +2,12 @@ import { ConnectorRegistry } from "../models/ConnectorRegistry"; class ConnectorService { - findConnectors = async (where?: Record, attributes?: string[]): Promise => { + findConnectors = (where?: Record, attributes?: string[]): Promise => { return ConnectorRegistry.findAll({ where, attributes, raw: true }); } - getConnector = async (where?: Record, attributes?: string[]): Promise => { - return ConnectorRegistry.findOne({ where, attributes, raw: true}); + getConnector = (where?: Record, attributes?: string[]): Promise => { + return ConnectorRegistry.findOne({ where, attributes, raw: true }); } } From 3da564f9a3eb323a64332353b6711eed3ca7d9d0 Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 1 Aug 2024 15:42:18 +0530 Subject: [PATCH 108/235] #OBS-I138: removed datakey before merging defaults to dataset --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 60332abb..375bb58b 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -95,7 +95,10 @@ const readyForPublish = async (dataset: Record) => { let draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) - _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue) => { + if (draftDataset?.type === 'master') { + defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); + } + _.mergeWith(draftDataset,defaultConfigs,draftDataset, (objValue, srcValue) => { if (_.isBoolean(objValue) && _.isBoolean(srcValue)) { return objValue; } From bed47984646f1eda90ba94b8e5afa13f2a504d14 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 1 Aug 2024 15:56:22 +0530 Subject: [PATCH 109/235] #OBS-I145: updated postman collection and swagger documentation --- .../updated_v2_collection.json | 53 +- .../swagger-doc/v2_updated_doc_openapi.yml | 465 ++++++++---------- 2 files changed, 213 insertions(+), 305 deletions(-) diff --git a/api-service/postman-collection/updated_v2_collection.json b/api-service/postman-collection/updated_v2_collection.json index 0552c08d..22412089 100644 --- a/api-service/postman-collection/updated_v2_collection.json +++ b/api-service/postman-collection/updated_v2_collection.json @@ -2589,12 +2589,12 @@ "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:17:54+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"7587f564-c2d7-49a8-9e56-dc56f6808ced\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"postgres-connector-1.0.0\",\n \"connector_id\": \"postgres-connector\",\n \"name\": \"PostgreSQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"1.0.0\",\n \"description\": \"The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg\",\n \"status\": \"Live\",\n \"ui_spec\": {\n \"schema\": {\n \"type\": \"object\",\n \"properties\": {\n \"connector_config\": {\n \"title\": \"Connector Config\",\n \"type\": \"object\",\n \"encrypt\": true,\n \"properties\": {\n \"databaseType\": {\n \"type\": \"string\",\n \"title\": \"Database Type\",\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"databaseType\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"databaseType\": {\n \"enum\": [\n \"PostgreSQL\",\n \"MySQL\"\n ]\n },\n \"connection_info\": {\n \"title\": \"Connection Information\",\n \"type\": \"object\",\n \"properties\": {\n \"host\": {\n \"type\": \"string\",\n \"title\": \"Database Host\",\n \"pattern\": \"/^(?:[a-zA-Z0-9-]+\\\\.)+[a-zA-Z]{2,}(?:/[^\\\\s]*)?|localhost(?:/[^\\\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"port\": {\n \"type\": \"number\",\n \"title\": \"Database Port\",\n \"minimum\": 1,\n \"maximum\": 65535,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"name\": {\n \"type\": \"string\",\n \"title\": \"Database Name\",\n \"pattern\": \"^[a-zA-Z0-9_]{1,64}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"username\": {\n \"type\": \"string\",\n \"title\": \"Database Username\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"password\": {\n \"type\": \"string\",\n \"title\": \"Database Password\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n },\n \"schemaInfo\": {\n \"title\": \"Schema Information\",\n \"type\": \"object\",\n \"properties\": {\n \"table\": {\n \"title\": \"Table Name\",\n \"type\": \"string\",\n \"pattern\": \"^[a-zA-Z_][a-zA-Z0-9_]{0,62}$\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"timestampColumn\": {\n \"title\": \"Timestamp Column\",\n \"type\": \"string\",\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n }\n }\n }\n }\n ]\n }\n }\n },\n \"operations_config\": {\n \"title\": \"Operations Configuration\",\n \"type\": \"object\",\n \"properties\": {\n \"batch_size\": {\n \"type\": \"number\",\n \"title\": \"Batch Size\",\n \"default\": 100,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"max_batches\": {\n \"type\": \"number\",\n \"title\": \"Maximum Batches\",\n \"default\": 10,\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n },\n \"pollingInterval\": {\n \"type\": \"string\",\n \"title\": \"Polling Interval\",\n \"enum\": [\n \"Once\",\n \"Periodic\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"Select polling interval\"\n }\n ]\n }\n },\n \"dependencies\": {\n \"pollingInterval\": {\n \"oneOf\": [\n {\n \"properties\": {\n \"pollingInterval\": {\n \"enum\": [\n \"Periodic\"\n ]\n },\n \"schedule\": {\n \"type\": \"string\",\n \"title\": \"Schedule\",\n \"enum\": [\n \"Hourly\",\n \"Daily\",\n \"Weekly\",\n \"Monthly\"\n ],\n \"fieldDescription\": [\n {\n \"type\": \"string\",\n \"description\": \"\"\n }\n ]\n }\n },\n \"required\": [\n \"schedule\"\n ]\n }\n ]\n }\n }\n }\n }\n },\n \"properties\": {\n \"connector_config\": {\n \"connection_info\": {\n \"password\": {\n \"ui:widget\": \"password\"\n }\n }\n },\n \"operations_config\": {\n \"batch_size\": {\n \"ui:readonly\": true\n },\n \"max_batches\": {\n \"ui:readonly\": true\n }\n }\n }\n },\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.732Z\",\n \"updated_date\": \"2024-06-25T04:38:28.732Z\",\n \"live_date\": \"2024-06-25T04:38:28.732Z\"\n }\n}" }, { - "name": "Failure: Connector not found", + "name": "Success: Read Draft Connector", "originalRequest": { "method": "GET", "header": [], "url": { - "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0?mode=edit", + "raw": "http://localhost:3000/v2/connectors/read/mssql-connector-2.0.0?mode=edit", "protocol": "http", "host": [ "localhost" @@ -2604,7 +2604,7 @@ "v2", "connectors", "read", - "postgres-connector-1.0.0" + "mssql-connector-2.0.0" ], "query": [ { @@ -2614,8 +2614,8 @@ ] } }, - "status": "Not Found", - "code": 404, + "status": "OK", + "code": 200, "_postman_previewlanguage": "json", "header": [ { @@ -2628,15 +2628,15 @@ }, { "key": "Content-Length", - "value": "305" + "value": "744" }, { "key": "ETag", - "value": "W/\"131-34hIJjn0HC4sZfrhBno7pcto+x4\"" + "value": "W/\"2e8-ZECTAoupjwTfEqQmuPSIRUFjF4o\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 12:49:17 GMT" + "value": "Thu, 01 Aug 2024 07:17:12 GMT" }, { "key": "Connection", @@ -2648,36 +2648,17 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:19:17+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"c3333acf-761e-4601-8254-850fd8641494\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector with the given id: postgres-connector-1.0.0 not found\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T12:47:12+05:30\",\n \"params\": {\n \"status\": \"SUCCESS\",\n \"resmsgid\": \"b6fcfb05-246c-4a1b-9eb1-27497ee9b80b\"\n },\n \"responseCode\": \"OK\",\n \"result\": {\n \"id\": \"mssql-connector-2.0.0\",\n \"connector_id\": \"mssql-connector\",\n \"name\": \"MS SQL\",\n \"type\": \"source\",\n \"category\": \"Database\",\n \"version\": \"2.0.0\",\n \"description\": \"The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform\",\n \"licence\": \"MIT\",\n \"owner\": \"Sunbird\",\n \"iconurl\": \"https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg\",\n \"status\": \"Draft\",\n \"ui_spec\": {},\n \"created_by\": \"SYSTEM\",\n \"updated_by\": \"SYSTEM\",\n \"created_date\": \"2024-06-25T04:38:28.847Z\",\n \"updated_date\": \"2024-06-25T04:38:28.847Z\",\n \"live_date\": \"2024-06-25T04:38:28.847Z\"\n }\n}" }, { - "name": "Failure: Invalid mode value", + "name": "Failure: Connector not found", "originalRequest": { "method": "GET", "header": [], - "url": { - "raw": "http://localhost:3000/v2/connectors/read/postgres-connector-1.0.0?mode=23", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "3000", - "path": [ - "v2", - "connectors", - "read", - "postgres-connector-1.0.0" - ], - "query": [ - { - "key": "mode", - "value": "23" - } - ] - } + "url": "http://localhost:3000/v2/connectors/read/postgres-conn" }, - "status": "Bad Request", - "code": 400, + "status": "Not Found", + "code": 404, "_postman_previewlanguage": "json", "header": [ { @@ -2690,15 +2671,15 @@ }, { "key": "Content-Length", - "value": "307" + "value": "276" }, { "key": "ETag", - "value": "W/\"133-NqsLTfEXPavK/auClQsE69MITWw\"" + "value": "W/\"114-izVC8DsHdeSfau/USVJvnIqZIMQ\"" }, { "key": "Date", - "value": "Wed, 31 Jul 2024 13:17:54 GMT" + "value": "Thu, 01 Aug 2024 09:32:48 GMT" }, { "key": "Connection", @@ -2710,7 +2691,7 @@ } ], "cookie": [], - "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-07-31T18:47:54+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"5ce8b69d-cda2-4035-8b82-d8b64cf3c574\"\n },\n \"responseCode\": \"BAD_REQUEST\",\n \"result\": {},\n \"error\": {\n \"code\": \"DATASET_INVALID_MODE_VALUE\",\n \"message\": \"The specified mode [23] in the query param is not valid.\"\n }\n}" + "body": "{\n \"id\": \"api.connectors.read\",\n \"ver\": \"v2\",\n \"ts\": \"2024-08-01T15:02:48+05:30\",\n \"params\": {\n \"status\": \"FAILED\",\n \"resmsgid\": \"712e7298-99f8-4694-9011-4232fcfd664a\"\n },\n \"responseCode\": \"NOT_FOUND\",\n \"result\": {},\n \"error\": {\n \"code\": \"CONNECTOR_NOT_FOUND\",\n \"message\": \"Connector not found: postgres-conn\"\n }\n}" } ] } diff --git a/api-service/swagger-doc/v2_updated_doc_openapi.yml b/api-service/swagger-doc/v2_updated_doc_openapi.yml index c95a15a7..c5f234a1 100644 --- a/api-service/swagger-doc/v2_updated_doc_openapi.yml +++ b/api-service/swagger-doc/v2_updated_doc_openapi.yml @@ -4668,284 +4668,213 @@ paths: responses: '200': description: OK - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '3304' - ETag: - schema: - type: string - example: W/"ce8-fwSqHq6/kVRau9kWO0rqLFp9a28" - Date: - schema: - type: string - example: Wed, 31 Jul 2024 12:47:54 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: type: object - example: - id: api.connectors.read - ver: v2 - ts: '2024-07-31T18:17:54+05:30' - params: - status: SUCCESS - resmsgid: 7587f564-c2d7-49a8-9e56-dc56f6808ced - responseCode: OK - result: - id: postgres-connector-1.0.0 - connector_id: postgres-connector - name: PostgreSQL - type: source - category: Database - version: 1.0.0 - description: >- - The PostgreSQL Connector is used to move data from any - Postgres Table to the Obsrv platform - licence: MIT - owner: Sunbird - iconurl: >- - https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg - status: Live - ui_spec: - schema: - type: object - properties: - connector_config: - title: Connector Config + examples: + example-0: + summary: 'Success: Read live Connector' + value: + id: api.connectors.read + ver: v2 + ts: '2024-07-31T18:17:54+05:30' + params: + status: SUCCESS + resmsgid: 7587f564-c2d7-49a8-9e56-dc56f6808ced + responseCode: OK + result: + id: postgres-connector-1.0.0 + connector_id: postgres-connector + name: PostgreSQL + type: source + category: Database + version: 1.0.0 + description: >- + The PostgreSQL Connector is used to move data from any + Postgres Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg + status: Live + ui_spec: + schema: type: object - encrypt: true properties: - databaseType: - type: string - title: Database Type - enum: - - PostgreSQL - - MySQL - fieldDescription: - - type: string - description: '' - dependencies: - databaseType: - oneOf: - - properties: - databaseType: - enum: - - PostgreSQL - - MySQL - connection_info: - title: Connection Information - type: object - properties: - host: - type: string - title: Database Host - pattern: >- - /^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(?:/[^\s]*)?|localhost(?:/[^\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/ - fieldDescription: - - type: string - description: '' - port: - type: number - title: Database Port - minimum: 1 - maximum: 65535 - fieldDescription: - - type: string - description: '' - name: - type: string - title: Database Name - pattern: ^[a-zA-Z0-9_]{1,64}$ - fieldDescription: - - type: string - description: '' - username: - type: string - title: Database Username - fieldDescription: - - type: string - description: '' - password: - type: string - title: Database Password - fieldDescription: - - type: string - description: '' - schemaInfo: - title: Schema Information - type: object - properties: - table: - title: Table Name - type: string - pattern: ^[a-zA-Z_][a-zA-Z0-9_]{0,62}$ - fieldDescription: - - type: string - description: '' - timestampColumn: - title: Timestamp Column + connector_config: + title: Connector Config + type: object + encrypt: true + properties: + databaseType: + type: string + title: Database Type + enum: + - PostgreSQL + - MySQL + fieldDescription: + - type: string + description: '' + dependencies: + databaseType: + oneOf: + - properties: + databaseType: + enum: + - PostgreSQL + - MySQL + connection_info: + title: Connection Information + type: object + properties: + host: + type: string + title: Database Host + pattern: >- + /^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}(?:/[^\s]*)?|localhost(?:/[^\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/ + fieldDescription: + - type: string + description: '' + port: + type: number + title: Database Port + minimum: 1 + maximum: 65535 + fieldDescription: + - type: string + description: '' + name: + type: string + title: Database Name + pattern: ^[a-zA-Z0-9_]{1,64}$ + fieldDescription: + - type: string + description: '' + username: + type: string + title: Database Username + fieldDescription: + - type: string + description: '' + password: + type: string + title: Database Password + fieldDescription: + - type: string + description: '' + schemaInfo: + title: Schema Information + type: object + properties: + table: + title: Table Name + type: string + pattern: ^[a-zA-Z_][a-zA-Z0-9_]{0,62}$ + fieldDescription: + - type: string + description: '' + timestampColumn: + title: Timestamp Column + type: string + fieldDescription: + - type: string + description: '' + operations_config: + title: Operations Configuration + type: object + properties: + batch_size: + type: number + title: Batch Size + default: 100 + fieldDescription: + - type: string + description: '' + max_batches: + type: number + title: Maximum Batches + default: 10 + fieldDescription: + - type: string + description: '' + pollingInterval: + type: string + title: Polling Interval + enum: + - Once + - Periodic + fieldDescription: + - type: string + description: Select polling interval + dependencies: + pollingInterval: + oneOf: + - properties: + pollingInterval: + enum: + - Periodic + schedule: type: string + title: Schedule + enum: + - Hourly + - Daily + - Weekly + - Monthly fieldDescription: - type: string description: '' - operations_config: - title: Operations Configuration - type: object - properties: + required: + - schedule + properties: + connector_config: + connection_info: + password: + ui:widget: password + operations_config: batch_size: - type: number - title: Batch Size - default: 100 - fieldDescription: - - type: string - description: '' + ui:readonly: true max_batches: - type: number - title: Maximum Batches - default: 10 - fieldDescription: - - type: string - description: '' - pollingInterval: - type: string - title: Polling Interval - enum: - - Once - - Periodic - fieldDescription: - - type: string - description: Select polling interval - dependencies: - pollingInterval: - oneOf: - - properties: - pollingInterval: - enum: - - Periodic - schedule: - type: string - title: Schedule - enum: - - Hourly - - Daily - - Weekly - - Monthly - fieldDescription: - - type: string - description: '' - required: - - schedule - properties: - connector_config: - connection_info: - password: - ui:widget: password - operations_config: - batch_size: - ui:readonly: true - max_batches: - ui:readonly: true - created_by: SYSTEM - updated_by: SYSTEM - created_date: '2024-06-25T04:38:28.732Z' - updated_date: '2024-06-25T04:38:28.732Z' - live_date: '2024-06-25T04:38:28.732Z' - '400': - description: Bad Request - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '307' - ETag: - schema: - type: string - example: W/"133-NqsLTfEXPavK/auClQsE69MITWw" - Date: - schema: - type: string - example: Wed, 31 Jul 2024 13:17:54 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 - content: - application/json: - schema: - type: object - example: - id: api.connectors.read - ver: v2 - ts: '2024-07-31T18:47:54+05:30' - params: - status: FAILED - resmsgid: 5ce8b69d-cda2-4035-8b82-d8b64cf3c574 - responseCode: BAD_REQUEST - result: {} - error: - code: DATASET_INVALID_MODE_VALUE - message: The specified mode [23] in the query param is not valid. + ui:readonly: true + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.732Z' + updated_date: '2024-06-25T04:38:28.732Z' + live_date: '2024-06-25T04:38:28.732Z' + example-1: + summary: 'Success: Read Draft Connector' + value: + id: api.connectors.read + ver: v2 + ts: '2024-08-01T12:47:12+05:30' + params: + status: SUCCESS + resmsgid: b6fcfb05-246c-4a1b-9eb1-27497ee9b80b + responseCode: OK + result: + id: mssql-connector-2.0.0 + connector_id: mssql-connector + name: MS SQL + type: source + category: Database + version: 2.0.0 + description: >- + The MS SQL Connector is used to move data from any MS + SQL Table to the Obsrv platform + licence: MIT + owner: Sunbird + iconurl: >- + https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg + status: Draft + ui_spec: {} + created_by: SYSTEM + updated_by: SYSTEM + created_date: '2024-06-25T04:38:28.847Z' + updated_date: '2024-06-25T04:38:28.847Z' + live_date: '2024-06-25T04:38:28.847Z' '404': description: Not Found - headers: - X-Powered-By: - schema: - type: string - example: Express - Content-Type: - schema: - type: string - example: application/json; charset=utf-8 - Content-Length: - schema: - type: integer - example: '305' - ETag: - schema: - type: string - example: W/"131-34hIJjn0HC4sZfrhBno7pcto+x4" - Date: - schema: - type: string - example: Wed, 31 Jul 2024 12:49:17 GMT - Connection: - schema: - type: string - example: keep-alive - Keep-Alive: - schema: - type: string - example: timeout=5 content: application/json: schema: @@ -4953,14 +4882,12 @@ paths: example: id: api.connectors.read ver: v2 - ts: '2024-07-31T18:19:17+05:30' + ts: '2024-08-01T15:02:48+05:30' params: status: FAILED - resmsgid: c3333acf-761e-4601-8254-850fd8641494 + resmsgid: 712e7298-99f8-4694-9011-4232fcfd664a responseCode: NOT_FOUND result: {} error: code: CONNECTOR_NOT_FOUND - message: >- - Connector with the given id: postgres-connector-1.0.0 not - found \ No newline at end of file + message: 'Connector not found: postgres-conn' \ No newline at end of file From cb46f9bd163c363e9be7b64cdfbbf401fbe477ab Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 1 Aug 2024 16:52:54 +0530 Subject: [PATCH 110/235] #OBS-I145: added the test cases for connector read api --- .../ConnectorsRead/ConnectorsRead.spec.ts | 98 ++++++ .../Connectors/ConnectorsRead/Fixtures.ts | 284 ++++++++++++++++++ 2 files changed, 382 insertions(+) create mode 100644 api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts create mode 100644 api-service/src/tests/Connectors/ConnectorsRead/Fixtures.ts diff --git a/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts b/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts new file mode 100644 index 00000000..f7fc8030 --- /dev/null +++ b/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts @@ -0,0 +1,98 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import { ConnectorRegistry } from "../../../models/ConnectorRegistry"; +import { TestInputsForConnectorsRead } from "./Fixtures"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +const apiId = "api.connectors.read" + +describe("Connectors Read API", () => { + afterEach(() => { + chai.spy.restore(); + }) + + it("Connector read success: When mode is not provided", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve(TestInputsForConnectorsRead.LIVE_CONNECTORS) + }) + chai + .request(app) + .get("/v2/connectors/read/postgres-connector-1.0.0") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForConnectorsRead.LIVE_CONNECTORS)) + done(); + }); + }); + + it("Connector read success: With mode=edit", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve(TestInputsForConnectorsRead.DRAFT_CONNECTORS) + }) + chai + .request(app) + .get("/v2/connectors/read/mssql-connector-2.0.0?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForConnectorsRead.DRAFT_CONNECTORS)) + done(); + }); + }); + + it("Connector read failure: when the requested connector of the id is not found", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve() + }) + chai + .request(app) + .get("/v2/connectors/read/postgres-connector-1.0.0?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("Connector not found: postgres-connector-1.0.0") + res.body.error.code.should.be.eq("CONNECTOR_NOT_FOUND") + res.should.have.status(httpStatus.NOT_FOUND); + done(); + }); + }) + + + + it("Connector read Failure: when invalid id provided", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve() + }) + chai + .request(app) + .get("/v2/connectors/read/postgres-conn") + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("Connector not found: postgres-conn") + res.body.error.code.should.be.eq("CONNECTOR_NOT_FOUND") + res.should.have.status(httpStatus.NOT_FOUND); + done(); + }); + }); +}) diff --git a/api-service/src/tests/Connectors/ConnectorsRead/Fixtures.ts b/api-service/src/tests/Connectors/ConnectorsRead/Fixtures.ts new file mode 100644 index 00000000..cb0c504f --- /dev/null +++ b/api-service/src/tests/Connectors/ConnectorsRead/Fixtures.ts @@ -0,0 +1,284 @@ +export const TestInputsForConnectorsRead = { + LIVE_CONNECTORS: { + "id": "api.connectors.read", + "ver": "v2", + "ts": "2024-07-31T18:17:54+05:30", + "params": { + "status": "SUCCESS", + "resmsgid": "7587f564-c2d7-49a8-9e56-dc56f6808ced" + }, + "responseCode": "OK", + "result": { + "id": "postgres-connector-1.0.0", + "connector_id": "postgres-connector", + "name": "PostgreSQL", + "type": "source", + "category": "Database", + "version": "1.0.0", + "description": "The PostgreSQL Connector is used to move data from any Postgres Table to the Obsrv platform", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/2/29/Postgresql_elephant.svg", + "status": "Live", + "ui_spec": { + "schema": { + "type": "object", + "properties": { + "connector_config": { + "title": "Connector Config", + "type": "object", + "encrypt": true, + "properties": { + "databaseType": { + "type": "string", + "title": "Database Type", + "enum": [ + "PostgreSQL", + "MySQL" + ], + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + } + }, + "dependencies": { + "databaseType": { + "oneOf": [ + { + "properties": { + "databaseType": { + "enum": [ + "PostgreSQL", + "MySQL" + ] + }, + "connection_info": { + "title": "Connection Information", + "type": "object", + "properties": { + "host": { + "type": "string", + "title": "Database Host", + "pattern": "/^(?:[a-zA-Z0-9-]+\\.)+[a-zA-Z]{2,}(?:/[^\\s]*)?|localhost(?:/[^\\s]*)?|((?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)/", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "port": { + "type": "number", + "title": "Database Port", + "minimum": 1, + "maximum": 65535, + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "name": { + "type": "string", + "title": "Database Name", + "pattern": "^[a-zA-Z0-9_]{1,64}$", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "username": { + "type": "string", + "title": "Database Username", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "password": { + "type": "string", + "title": "Database Password", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + } + } + }, + "schemaInfo": { + "title": "Schema Information", + "type": "object", + "properties": { + "table": { + "title": "Table Name", + "type": "string", + "pattern": "^[a-zA-Z_][a-zA-Z0-9_]{0,62}$", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "timestampColumn": { + "title": "Timestamp Column", + "type": "string", + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + } + } + } + } + } + ] + } + } + }, + "operations_config": { + "title": "Operations Configuration", + "type": "object", + "properties": { + "batch_size": { + "type": "number", + "title": "Batch Size", + "default": 100, + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "max_batches": { + "type": "number", + "title": "Maximum Batches", + "default": 10, + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + }, + "pollingInterval": { + "type": "string", + "title": "Polling Interval", + "enum": [ + "Once", + "Periodic" + ], + "fieldDescription": [ + { + "type": "string", + "description": "Select polling interval" + } + ] + } + }, + "dependencies": { + "pollingInterval": { + "oneOf": [ + { + "properties": { + "pollingInterval": { + "enum": [ + "Periodic" + ] + }, + "schedule": { + "type": "string", + "title": "Schedule", + "enum": [ + "Hourly", + "Daily", + "Weekly", + "Monthly" + ], + "fieldDescription": [ + { + "type": "string", + "description": "" + } + ] + } + }, + "required": [ + "schedule" + ] + } + ] + } + } + } + } + }, + "properties": { + "connector_config": { + "connection_info": { + "password": { + "ui:widget": "password" + } + } + }, + "operations_config": { + "batch_size": { + "ui:readonly": true + }, + "max_batches": { + "ui:readonly": true + } + } + } + }, + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.732Z", + "updated_date": "2024-06-25T04:38:28.732Z", + "live_date": "2024-06-25T04:38:28.732Z" + } + }, + + DRAFT_CONNECTORS: { + "id": "api.connectors.read", + "ver": "v2", + "ts": "2024-08-01T12:47:12+05:30", + "params": { + "status": "SUCCESS", + "resmsgid": "b6fcfb05-246c-4a1b-9eb1-27497ee9b80b" + }, + "responseCode": "OK", + "result": { + "id": "mssql-connector-2.0.0", + "connector_id": "mssql-connector", + "name": "MS SQL", + "type": "source", + "category": "Database", + "version": "2.0.0", + "description": "The MS SQL Connector is used to move data from any MS SQL Table to the Obsrv platform", + "licence": "MIT", + "owner": "Sunbird", + "iconurl": "https://upload.wikimedia.org/wikipedia/commons/2/29/Microsoft_SQL_Server_Logo.svg", + "status": "Draft", + "ui_spec": {}, + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-06-25T04:38:28.847Z", + "updated_date": "2024-06-25T04:38:28.847Z", + "live_date": "2024-06-25T04:38:28.847Z" + } + } +} \ No newline at end of file From 9c32800a206827f5bab651c5d6522df94597ed80 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 1 Aug 2024 16:59:49 +0530 Subject: [PATCH 111/235] #OBS-I145: added a test case for connector read api --- .../ConnectorsRead/ConnectorsRead.spec.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts b/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts index f7fc8030..e3247e3d 100644 --- a/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts +++ b/api-service/src/tests/Connectors/ConnectorsRead/ConnectorsRead.spec.ts @@ -37,6 +37,25 @@ describe("Connectors Read API", () => { }); }); + it("Connector read success: When valid id is given, but value of mode is not provided", (done) => { + chai.spy.on(ConnectorRegistry, "findOne", () => { + return Promise.resolve(TestInputsForConnectorsRead.LIVE_CONNECTORS) + }) + chai + .request(app) + .get("/v2/connectors/read/postgres-connector-1.0.0?mode=") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(TestInputsForConnectorsRead.LIVE_CONNECTORS)) + done(); + }); + }); + it("Connector read success: With mode=edit", (done) => { chai.spy.on(ConnectorRegistry, "findOne", () => { return Promise.resolve(TestInputsForConnectorsRead.DRAFT_CONNECTORS) From 22d6cb2a76dc8fc625e0ff496828f729357fdf0d Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Sun, 4 Aug 2024 14:44:28 +0530 Subject: [PATCH 112/235] #OBS-I116: feat: Dataset import api fixes --- api-service/src/controllers/DatasetImport/DatasetImport.ts | 2 ++ .../src/controllers/DatasetImport/DatasetImportHelper.ts | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 3f5ef29f..d916ed2b 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -31,6 +31,8 @@ const importDataset = async (dataset: Record, overwrite: string | a throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) return _.omit(overwriteRes, ["message"]) + } else { + throw obsrvError(dataset_id, "DATASET_EXISTS", `Dataset with dataset_id: ${dataset_id} already exists.`, "CONFLICT", 409); } } if(response?.errors){ diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts index 9388ab21..849e4fdd 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -50,12 +50,13 @@ export const datasetImportValidation = async (payload: Record): Pro datasetConfig["denorm_config"] = { ...denormConfig, denorm_fields: resultantDenorms }; datasetConfig["router_config"] = { topic: datasetConfig.id } datasetConfig["version_key"] = Date.now().toString() + datasetConfig["status"] = DatasetStatus.Draft const defaults = _.cloneDeep(defaultDatasetConfig); const resultantDataset = _.merge(defaults, datasetConfig); return { - updatedDataset: _.omit(resultantDataset, ["created_date", "updated_date", "published_date", "status"]), + updatedDataset: _.omit(resultantDataset, ["created_date", "updated_date", "published_date"]), ignoredFields: { ignoredConnectors, ignoredTransformations, ignoredDenorms: [...ignoredDenorms, ...invalidDenorms] } }; }; From 0c3edb7a4f1657973dfa8025e9444eb81d6b46d0 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 5 Aug 2024 11:25:39 +0530 Subject: [PATCH 113/235] #OBS-I116: feat: Test cases and linting fixes --- .github/workflows/pull_request.yaml | 1 - api-service/.eslintignore | 2 +- api-service/package.json | 5 +-- api-service/src/configs/Config.ts | 2 +- .../src/connections/grafanaConnection.ts | 2 +- api-service/src/controllers/Alerts/Alerts.ts | 16 +++---- api-service/src/controllers/Alerts/Metric.ts | 20 ++++----- api-service/src/controllers/Alerts/Silence.ts | 18 ++++---- .../controllers/DatasetCopy/DatasetCopy.ts | 6 +-- .../DatasetCopy/DatasetCopyHelper.ts | 6 +-- .../DatasetImport/DatasetImport.ts | 6 +-- .../DatasetImport/DatasetImportHelper.ts | 4 +- .../controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition.ts | 12 +++--- .../NotificationChannel/Notification.ts | 20 ++++----- api-service/src/helpers/ResponseHandler.ts | 2 +- api-service/src/models/Alert.ts | 2 +- api-service/src/models/Metric.ts | 2 +- api-service/src/models/Notification.ts | 2 +- api-service/src/models/Silence.ts | 2 +- api-service/src/routes/AlertsRouter.ts | 2 +- .../src/services/DatasetHealthService.ts | 6 +-- api-service/src/services/DatasetService.ts | 6 +-- api-service/src/services/HealthService.ts | 4 +- api-service/src/services/WrapperService.ts | 1 - api-service/src/services/fs.ts | 6 +-- .../managers/grafana/alert/helpers/index.ts | 18 ++++---- .../services/managers/grafana/alert/index.ts | 26 ++++++------ .../src/services/managers/grafana/index.ts | 6 +-- .../grafana/notification/channels/email.ts | 8 ++-- .../grafana/notification/channels/index.ts | 8 ++-- .../grafana/notification/channels/slack.ts | 7 ++-- .../grafana/notification/channels/teams.ts | 7 ++-- .../grafana/notification/helpers/index.ts | 20 ++++----- .../managers/grafana/notification/index.ts | 3 +- .../grafana/silences/helpers/index.ts | 2 +- api-service/src/services/managers/index.ts | 14 +++---- .../managers/prometheus/alert/index.ts | 12 +++--- .../src/services/managers/prometheus/index.ts | 6 +-- .../managers/prometheus/notification/index.ts | 8 ++-- .../managers/prometheus/silences/index.ts | 10 ++--- .../DataIngestTest/DataIngestionTest.spec.ts | 19 ++++----- .../DataOutTest/DataQueryTest.spec.ts | 8 ++-- .../DatasetManagement/DataOutTest/Fixtures.ts | 22 +++++----- .../DatasetCreate/DatasetCreate.spec.ts | 3 +- .../DatasetCreate/Fixtures.ts | 1 - .../DatasetList/DatasetList.spec.ts | 5 +-- .../DatasetRead/DatasetRead.spec.ts | 20 ++++----- .../DatasetDelete.spec.ts | 4 +- .../DatasetLive.spec.ts | 4 +- .../DatasetReadyToPublish.spec.ts | 4 +- .../DatasetRetire.spec.ts | 4 +- .../DatasetStatusTransition.spec.ts | 4 +- .../DatasetUpdate/DatasetDedup.spec.ts | 4 +- .../DatasetUpdate/DatasetDenorm.spec.ts | 2 +- .../DatasetUpdate/DatasetExtraction.spec.ts | 4 +- .../DatasetUpdate/DatasetTags.spec.ts | 2 +- .../DatasetTransformation.spec.ts | 2 +- .../DatasetUpdate/DatasetUpdate.spec.ts | 2 +- .../DatasetUpdate/DatasetValidation.spec.ts | 4 +- .../GenerateSignedURL/Fixtures.ts | 16 +++---- .../GenerateSignedURL.spec.ts | 4 +- .../CreateTemplate/CreateTemplate.spec.ts | 6 +-- .../DeleteTemplate/DeleteTemplate.spec.ts | 8 ++-- .../ListTemplates/ListTemplates.spec.ts | 6 +-- .../ReadTemplate/ReadTemplate.spec.ts | 18 ++++---- .../TemplateQuerying/TemplateQuerying.spec.ts | 42 +++++++++---------- .../UpdateTemplate/UpdateTemplate.spec.ts | 6 +-- .../SqlWrapper/SqlWrapper.spec.ts | 4 +- 69 files changed, 265 insertions(+), 275 deletions(-) diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 12e967e2..8b4fc54d 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -24,5 +24,4 @@ jobs: cd api-service npm install npm run actions:test - npm run actions:test:v2 npm run lint \ No newline at end of file diff --git a/api-service/.eslintignore b/api-service/.eslintignore index f56aadad..e528fbc0 100644 --- a/api-service/.eslintignore +++ b/api-service/.eslintignore @@ -4,6 +4,6 @@ docs coverage @types .nyc_output -src/v2/tests +src/tests src/v1 dist \ No newline at end of file diff --git a/api-service/package.json b/api-service/package.json index f760df95..1360ecf6 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -5,9 +5,8 @@ "main": "dist/app.js", "scripts": { "start": "ts-node ./src/app.ts", - "test": "source .env.test && nyc mocha ./src/v1/test/*.spec.ts --exit && nyc mocha ./src/v2/tests/**/*.spec.ts --exit", - "actions:test": "nyc mocha ./src/v1/test/*.spec.ts --exit", - "actions:test:v2": "nyc mocha ./src/v2/tests/**/*.spec.ts --exit", + "test": "source .env.test && nyc mocha ./src/tests/**/*.spec.ts --exit", + "actions:test": "nyc mocha ./src/test/*.spec.ts --exit", "build": "rm -rf dist && tsc --declaration -P . && cp package.json ./dist/package.json", "package": "npm run build && cd dist && npm pack . && cd ..", "lint": "eslint . --ext .ts", diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 28f83b1e..38fe624c 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -109,7 +109,7 @@ export const config = { "encryption_algorithm": process.env.encryption_algorithm || "aes-256-ecb", }, "grafana_config": { - "dialect": process.env.dialet || 'postgres', + "dialect": process.env.dialet || "postgres", "url": process.env.grafana_url || "http://localhost:8000", "access_token": process.env.grafana_token || "" } diff --git a/api-service/src/connections/grafanaConnection.ts b/api-service/src/connections/grafanaConnection.ts index 76df0cea..b70e6009 100644 --- a/api-service/src/connections/grafanaConnection.ts +++ b/api-service/src/connections/grafanaConnection.ts @@ -5,6 +5,6 @@ const grafanaHttpClient = axios.create({ baseURL: config.grafana_config.url }); -grafanaHttpClient.defaults.headers.common['Authorization'] = config.grafana_config.access_token; +grafanaHttpClient.defaults.headers.common["Authorization"] = config.grafana_config.access_token; export { grafanaHttpClient }; \ No newline at end of file diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 056a2475..278ac586 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -17,9 +17,9 @@ const createAlertHandler = async (req: Request, res: Response, next: NextFunctio updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); } catch (error: any) { - let errorMessage = _.get(error, 'message') - if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { - errorMessage = _.get(error, 'parent.detail') + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") } next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } @@ -44,7 +44,7 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const transformAlerts = async (alertModel: any) => { const alert = alertModel?.toJSON(); - const status = _.get(alert, 'status'); + const status = _.get(alert, "status"); if (status !== "live") return alert; return getAlertsMetadata(alert); } @@ -108,13 +108,13 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio await retireAlertSilence(alertId); } const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); - await Alert.update({ ...updatedPayload, status: 'draft' }, { where: { id: alertId } }); + await Alert.update({ ...updatedPayload, status: "draft" }, { where: { id: alertId } }); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); } catch (error: any) { - let errorMessage = _.get(error, 'message') - if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { - errorMessage = _.get(error, 'parent.detail') + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") } next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } diff --git a/api-service/src/controllers/Alerts/Metric.ts b/api-service/src/controllers/Alerts/Metric.ts index 212d5b13..6f80f16f 100644 --- a/api-service/src/controllers/Alerts/Metric.ts +++ b/api-service/src/controllers/Alerts/Metric.ts @@ -10,15 +10,15 @@ const telemetryObject = { type: "metric", ver: "1.0.0" }; const createMetricHandler = async (req: Request, res: Response, next: NextFunction) => { try { - const { component } = req?.body; + const { component } = req.body; const transformComponent = _.toLower(component); const metricsBody = await Metrics.create({ ...(req.body), component: transformComponent }); updateTelemetryAuditEvent({ request: req, object: { id: metricsBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: metricsBody.dataValues.id } }); } catch (error: any) { - let errorMessage = _.get(error, 'message') - if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { - errorMessage = _.get(error, 'parent.detail') + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") } next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } @@ -30,7 +30,7 @@ const listMetricsHandler = async (req: Request, res: Response, next: NextFunctio const metricsPayload = await Metrics.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }) }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { metrics: metricsPayload, count: metricsPayload.length } }); } catch (error) { - const errorMessage = _.get(error, 'message') + const errorMessage = _.get(error, "message") next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } } @@ -50,9 +50,9 @@ const updateMetricHandler = async (req: Request, res: Response, next: NextFuncti }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } }); } catch (error) { - let errorMessage = _.get(error, 'message') - if (_.get(error, 'name') == "SequelizeUniqueConstraintError") { - errorMessage = _.get(error, 'parent.detail') + let errorMessage = _.get(error, "message") + if (_.get(error, "name") == "SequelizeUniqueConstraintError") { + errorMessage = _.get(error, "parent.detail") } next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } @@ -66,7 +66,7 @@ const deleteMetricHandler = async (req: Request, res: Response, next: NextFuncti await record.destroy(); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id } }); } catch (error) { - const errorMessage = _.get(error, 'message') + const errorMessage = _.get(error, "message") next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } } @@ -78,7 +78,7 @@ const deleteMultipleMetricHandler = async (req: Request, res: Response, next: Ne await Metrics.destroy({ where: filters }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: {} }); } catch (error) { - const errorMessage = _.get(error, 'message') + const errorMessage = _.get(error, "message") next(errorResponse((httpStatus.INTERNAL_SERVER_ERROR, { message: errorMessage }))) } } diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index c578dcbc..32e2c531 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -18,8 +18,8 @@ const createHandler = async (request: Request, response: Response, next: NextFun const grafanaResponse = await createSilence(payload); if (!grafanaResponse) return next({ message: httpStatus[httpStatus.INTERNAL_SERVER_ERROR], statusCode: httpStatus.INTERNAL_SERVER_ERROR }) - let start_date = new Date(startDate); - let end_date = new Date(endDate); + const start_date = new Date(startDate); + const end_date = new Date(endDate); const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, @@ -31,7 +31,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: sileneResponse.dataValues.id } }) } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -45,11 +45,11 @@ const transformSilences = async (silenceModel: any) => { const listHandler = async (request: Request, response: Response, next: NextFunction) => { try { const silences = await Silence.findAll(); - const count = _.get(silences, 'length'); + const count = _.get(silences, "length"); const transformedSilences = await Promise.all(silences.map(transformSilences)); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { transformedSilences, ...(count && { count }) } }); } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -62,7 +62,7 @@ const fetchHandler = async (request: Request, response: Response, next: NextFunc if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: transformedSilence }); } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -86,7 +86,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun const silenceResponse = await Silence.update(updatedSilence, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } }) } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -97,13 +97,13 @@ const deleteHandler = async (request: Request, response: Response, next: NextFun const silenceModel = await Silence.findOne({ where: { id } }); if (!silenceModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); const silenceObject = silenceModel?.toJSON(); - if (silenceObject?.status === 'expired') return next({ message: "Silence is already expired", statusCode: httpStatus.BAD_REQUEST }); + if (silenceObject?.status === "expired") return next({ message: "Silence is already expired", statusCode: httpStatus.BAD_REQUEST }); await deleteSilence(silenceObject); await silenceModel.destroy(); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: silenceObject }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }) } catch (err) { - const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = errorResponse(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index b33b614e..9308dd71 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -18,7 +18,7 @@ const validateRequest = (req: Request) => { } } -const fetchDataset = async (req: Request, newDatasetId: string) => { +const fetchDataset = async (req: Request) => { const datasetId = _.get(req, "body.request.source.datasetId"); const isLive = _.get(req, "body.request.source.isLive"); @@ -39,10 +39,10 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); - const dataset = await fetchDataset(req, newDatasetId); + const dataset = await fetchDataset(req); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { - if (err?.name === 'SequelizeUniqueConstraintError') { + if (err?.name === "SequelizeUniqueConstraintError") { throw obsrvError(newDatasetId, "DATASET_ALREADY_EXISTS", `Dataset with id ${newDatasetId} already exists`, "BAD_REQUEST", 400); } throw obsrvError(newDatasetId, "DATASET_COPY_FAILURE", `Failed to clone dataset`, "INTERNAL_SERVER_ERROR", 500); diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts index de4c8087..7b50626c 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -6,13 +6,13 @@ const version = defaultDatasetConfig.version; export const updateRecords = (datasetRecord: Record, newDatasetId: string): void => { const dataset_id = newDatasetId; - _.set(datasetRecord, 'api_version', "v2") - _.set(datasetRecord, 'status', DatasetStatus.Draft) + _.set(datasetRecord, "api_version", "v2") + _.set(datasetRecord, "status", DatasetStatus.Draft) _.set(datasetRecord, "dataset_id", dataset_id) _.set(datasetRecord, "id", dataset_id) _.set(datasetRecord, "name", dataset_id) _.set(datasetRecord, "version_key", Date.now().toString()) - _.set(datasetRecord, 'version', version); + _.set(datasetRecord, "version", version); _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) _.set(datasetRecord, "router_config", { topic: newDatasetId }) } diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 3f5ef29f..e8a0b033 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -25,9 +25,9 @@ const datasetImport = async (req: Request, res: Response) => { const importDataset = async (dataset: Record, overwrite: string | any) => { const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) - if (response?.name === 'SequelizeUniqueConstraintError') { + if (response?.name === "SequelizeUniqueConstraintError") { if (overwrite === "true") { - const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(err=>{ + const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{ throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) return _.omit(overwriteRes, ["message"]) @@ -42,7 +42,7 @@ const importDataset = async (dataset: Record, overwrite: string | a const getResponseData = (ignoredConfigs: Record) => { const { ignoredConnectors, ignoredTransformations, ignoredDenorms } = ignoredConfigs; let successMsg = "Dataset is imported successfully"; - let partialIgnored: Record = {}; + const partialIgnored: Record = {}; if (ignoredConnectors.length || ignoredTransformations.length || ignoredDenorms.length) { successMsg = "Dataset is partially imported"; diff --git a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts index 9388ab21..b3ccb9bc 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImportHelper.ts @@ -34,7 +34,7 @@ export const datasetImportValidation = async (payload: Record): Pro throw obsrvError("", "DATASET_IMPORT_INVALID_CONFIGS", isRequestValid.message, "BAD_REQUEST", 400) } - let datasetConfig = payload.request; + const datasetConfig = payload.request; const connectors = _.get(datasetConfig, "connectors_config", []); const transformations = _.get(datasetConfig, "transformations_config", []); @@ -99,7 +99,7 @@ export const migrateExportedDatasetV1 = (requestPayload: Record) => const { dataset_id, timestamp_key = "", data_key = "", type: datasetType } = _.get(datasetPayload, "data.metadata") const type = datasetType === "master-dataset" ? DatasetType.master : DatasetType.event - let dataset: Record = { + const dataset: Record = { dataset_id, id: dataset_id, name: dataset_id, type, version_key: Date.now().toString(), api_version: "v2", diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 28909ff3..fc16b313 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -17,7 +17,7 @@ const validateRequest = (req: Request) => { const { dataset_id } = req.params; const fields = req.query.fields; - if (fields && typeof fields !== 'string') { + if (fields && typeof fields !== "string") { throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS_VAL", `The specified fields [${fields}] in the query param is not a string.`, "BAD_REQUEST", 400); } const fieldValues = fields ? _.split(fields, ",") : []; diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 05e42885..8506afd3 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -92,10 +92,10 @@ const deleteDataset = async (dataset: Record) => { const readyForPublish = async (dataset: Record) => { - let draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) + const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) - if (draftDataset?.type === 'master') { + if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } _.mergeWith(draftDataset,defaultConfigs,draftDataset, (objValue, srcValue) => { @@ -159,9 +159,9 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) } if(!_.isEmpty(md)){ if(md.api_version === "v2") - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); + datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); else - datasetStatus['denorm_field'] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); + datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); } return datasetStatus; @@ -187,8 +187,8 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) } const updateMasterDataConfig = async (draftDataset: Record) => { - if (draftDataset.type === 'master') { - let dataset_config = _.get(draftDataset, "dataset_config") + if (draftDataset.type === "master") { + const dataset_config = _.get(draftDataset, "dataset_config") const datasetCacheConfig = _.get(defaultDatasetConfig, "dataset_config.cache_config") draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } if (draftDataset.dataset_config.cache_config.redis_db === 0) { diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index 93c5a875..f6db4fd4 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -4,7 +4,7 @@ import httpStatus from "http-status"; import createError from "http-errors"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { publishNotificationChannel, testNotificationChannel, updateNotificationChannel } from "../../services/managers"; -import _ from 'lodash'; +import _ from "lodash"; import { updateTelemetryAuditEvent } from "../../services/telemetry"; const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; @@ -16,7 +16,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -29,13 +29,13 @@ const updateHandler = async (request: Request, response: Response, next: NextFun const notificationPayload = notificationPayloadModel?.toJSON(); if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); - if (_.get(notificationPayload, 'status') === "live") { + if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -44,10 +44,10 @@ const listHandler = async (request: Request, response: Response, next: NextFunct try { const { limit, filters, offset } = request.body?.request || {}; const notifications = await Notification.findAll({ limit: limit, offset: offset, ...(filters && { where: filters }) }); - const count = _.get(notifications, 'length'); + const count = _.get(notifications, "length"); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { notifications, ...(count && { count }) } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -61,7 +61,7 @@ const fetchHandler = async (request: Request, response: Response, next: NextFunc updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: notificationPayloadModel?.toJSON() }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -77,7 +77,7 @@ const retireHandler = async (request: Request, response: Response, next: NextFun await Notification.update({ status: "retired" }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -94,7 +94,7 @@ const publishHandler = async (request: Request, response: Response, next: NextFu Notification.update({ status: "live" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } @@ -114,7 +114,7 @@ const testNotifationChannelHandler = async (request: Request, response: Response } ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "Notification Sent" } }); } catch (err) { - const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, 'message') || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) + const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) next(error); } } diff --git a/api-service/src/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts index 3bac60b7..1713d4fa 100644 --- a/api-service/src/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -57,7 +57,7 @@ const ResponseHandler = { }, goneResponse: (req: Request, res: Response) => { - const { id, entity } = req as any; + const { id } = req as any; res.status(httpStatus.GONE).json({ id: id, ver: "v1", ts: Date.now(), params: { status: "FAILED", errmsg: "v1 APIs have been replace by /v2 APIs. Please refer to this link for more information" }, responseCode: httpStatus["410_NAME"] }) } } diff --git a/api-service/src/models/Alert.ts b/api-service/src/models/Alert.ts index 7f98cde5..e52952b8 100644 --- a/api-service/src/models/Alert.ts +++ b/api-service/src/models/Alert.ts @@ -1,6 +1,6 @@ import { DataTypes } from "sequelize"; import { sequelize } from "../connections/databaseConnection"; -import { v4 as uuidv4 } from 'uuid'; +import { v4 as uuidv4 } from "uuid"; export const Alert = sequelize.define("alerts", { id: { diff --git a/api-service/src/models/Metric.ts b/api-service/src/models/Metric.ts index 605ef291..ecc4cb27 100644 --- a/api-service/src/models/Metric.ts +++ b/api-service/src/models/Metric.ts @@ -1,6 +1,6 @@ import { DataTypes } from "sequelize"; import { sequelize } from "../connections/databaseConnection"; -import { v4 as uuidv4 } from 'uuid'; +import { v4 as uuidv4 } from "uuid"; export const Metrics = sequelize.define("metrics", { id: { diff --git a/api-service/src/models/Notification.ts b/api-service/src/models/Notification.ts index dc2bee5a..6203032c 100644 --- a/api-service/src/models/Notification.ts +++ b/api-service/src/models/Notification.ts @@ -1,6 +1,6 @@ import { DataTypes } from "sequelize"; import { sequelize } from "../connections/databaseConnection"; -import { v4 } from 'uuid'; +import { v4 } from "uuid"; export const Notification = sequelize.define("notificationchannel", { id: { diff --git a/api-service/src/models/Silence.ts b/api-service/src/models/Silence.ts index 3c272db1..d78d3f89 100644 --- a/api-service/src/models/Silence.ts +++ b/api-service/src/models/Silence.ts @@ -1,6 +1,6 @@ import { DataTypes } from "sequelize"; import { sequelize } from "../connections/databaseConnection"; -import { v4 as uuid } from 'uuid'; +import { v4 as uuid } from "uuid"; export const Silence = sequelize.define("silences", { id: { diff --git a/api-service/src/routes/AlertsRouter.ts b/api-service/src/routes/AlertsRouter.ts index 9efab170..01c843dd 100644 --- a/api-service/src/routes/AlertsRouter.ts +++ b/api-service/src/routes/AlertsRouter.ts @@ -1,5 +1,5 @@ import express from "express"; -import notificationHandler from '../controllers/NotificationChannel/Notification'; +import notificationHandler from "../controllers/NotificationChannel/Notification"; import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; import customAlertHandler from "../controllers/Alerts/Alerts"; import metricAliasHandler from "../controllers/Alerts/Metric"; diff --git a/api-service/src/services/DatasetHealthService.ts b/api-service/src/services/DatasetHealthService.ts index 92672c47..5ea8b6df 100644 --- a/api-service/src/services/DatasetHealthService.ts +++ b/api-service/src/services/DatasetHealthService.ts @@ -19,9 +19,9 @@ const prometheusQueries = { dedupFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_PipelinePreprocessorJob_DATASETID_dedup_failed_count[1d]))", denormFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_DenormalizerJob_DATASETID_denorm_failed[1d]))", transformationFailure: "sum(sum_over_time(flink_taskmanager_job_task_operator_TransformerJob_DATASETID_transform_failed_count[1d]))", - queriesCount: 'sum(sum_over_time(node_total_api_calls{entity="data-out", dataset_id="DATASETID"}[1d]))', - avgQueryResponseTimeInSec: 'avg(avg_over_time(node_query_response_time{entity="data-out", dataset_id="DATASETID"}[1d]))/1000', - queriesFailedCount: 'sum(sum_over_time(node_failed_api_calls{entity="data-out", dataset_id="DATASETID"}[1d]))' + queriesCount: "sum(sum_over_time(node_total_api_calls{entity=\"data-out\", dataset_id=\"DATASETID\"}[1d]))", + avgQueryResponseTimeInSec: "avg(avg_over_time(node_query_response_time{entity=\"data-out\", dataset_id=\"DATASETID\"}[1d]))/1000", + queriesFailedCount: "sum(sum_over_time(node_failed_api_calls{entity=\"data-out\", dataset_id=\"DATASETID\"}[1d]))" } export const getDatasetHealth = async (categories: any, dataset: any) => { diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 92db6058..8c6c1e62 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -115,7 +115,7 @@ class DatasetService { migrateDatasetV1 = async (dataset_id: string, dataset: Record): Promise => { const status = _.get(dataset, "status") - let draftDataset: Record = { + const draftDataset: Record = { api_version: "v2", version_key: Date.now().toString() } @@ -337,7 +337,7 @@ class DatasetService { const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); - _.set(draftDatasource, 'ingestion_spec', ingestionSpec) + _.set(draftDatasource, "ingestion_spec", ingestionSpec) await DatasourceDraft.create(draftDatasource, {transaction}) } @@ -357,7 +357,7 @@ class DatasetService { export const getLiveDatasetConfigs = async (dataset_id: string) => { - let datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) + const datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) diff --git a/api-service/src/services/HealthService.ts b/api-service/src/services/HealthService.ts index 273ceb00..c10a9ea6 100644 --- a/api-service/src/services/HealthService.ts +++ b/api-service/src/services/HealthService.ts @@ -1,9 +1,9 @@ -import { Request, Response, NextFunction } from "express" +import { Request, Response } from "express" import { ResponseHandler } from "../helpers/ResponseHandler" class HealthService { - async checkDruidHealth(req: Request, res: Response, next: NextFunction) { + async checkDruidHealth(req: Request, res: Response) { ResponseHandler.successResponse(req, res, { status: 200, data: {} }) } diff --git a/api-service/src/services/WrapperService.ts b/api-service/src/services/WrapperService.ts index 061b2b14..c38afb6d 100644 --- a/api-service/src/services/WrapperService.ts +++ b/api-service/src/services/WrapperService.ts @@ -1,6 +1,5 @@ import axios from "axios"; import { NextFunction, Request, Response } from "express"; -import _ from "lodash"; import { config } from "../configs/Config"; import { ResponseHandler } from "../helpers/ResponseHandler"; import { ErrorResponseHandler } from "../helpers/ErrorResponseHandler"; diff --git a/api-service/src/services/fs.ts b/api-service/src/services/fs.ts index e08597d6..774b87b9 100644 --- a/api-service/src/services/fs.ts +++ b/api-service/src/services/fs.ts @@ -1,8 +1,8 @@ -import fs from 'fs'; -import path from 'path'; +import fs from "fs"; +import path from "path"; export const scrapModules = (folderPath: string, basename: string) => { - const mapping = new Map>(); + const mapping = new Map>(); fs.readdirSync(folderPath) .filter((file) => file !== basename) .map((file) => { diff --git a/api-service/src/services/managers/grafana/alert/helpers/index.ts b/api-service/src/services/managers/grafana/alert/helpers/index.ts index e9c19e29..9c577a52 100644 --- a/api-service/src/services/managers/grafana/alert/helpers/index.ts +++ b/api-service/src/services/managers/grafana/alert/helpers/index.ts @@ -32,7 +32,7 @@ const deleteAlertFolder = async (folderName: string) => { const checkIfGroupNameExists = async (category: string) => { const response = await getRules(); - const rules = _.get(response, 'data'); + const rules = _.get(response, "data"); if(!_.has(rules, category)) return undefined; return _.find(_.flatMap(_.values(rules)), { name: category, @@ -59,7 +59,7 @@ const getSpecificRule = async (payload: Record) => { const alertrules = await alerts(); const groups = _.get(alertrules, "data.data.groups"); const ruleGroup = _.find(groups, (group: any) => group.name == payload.category); - return _.find(_.get(ruleGroup, 'rules'), (rule: any) => rule.name == payload.name); + return _.find(_.get(ruleGroup, "rules"), (rule: any) => rule.name == payload.name); }; const updateMetadata = (metadata: any, dataSource: string, expression: string) => { @@ -89,7 +89,7 @@ const createFolder = (title: string) => { const createFolderIfNotExists = async (folderName: string) => { const folders = await getFolders(); - const isExists = _.find(folders.data, folder => _.get(folder, 'title') === folderName); + const isExists = _.find(folders.data, folder => _.get(folder, "title") === folderName); if (isExists) return; return createFolder(folderName); } @@ -215,7 +215,7 @@ const queryOperators = [ const getQueryExpression = (payload: Record) => { const { metric, operator, threshold } = payload; - const operatorSymbol = _.get(_.find(queryOperators, operatorMetadata => _.get(operatorMetadata, 'value') === operator), 'symbol'); + const operatorSymbol = _.get(_.find(queryOperators, operatorMetadata => _.get(operatorMetadata, "value") === operator), "symbol"); return `(${metric}) ${operatorSymbol} ${threshold}`; } @@ -229,7 +229,7 @@ const getMatchingLabels = async (channels: string[]) => { const { name, type } = channelMetadata; return `notificationChannel_${name}_${type}`; }) - .catch(err => null); + .catch(() => null); } const matchingLabels = await Promise.all(channels.map(fetchChannel)); @@ -245,22 +245,22 @@ const getMatchingLabels = async (channels: string[]) => { const transformRule = async ({ value, condition, metadata, isGroup }: any) => { const { name, id, interval, category, frequency, labels = {}, annotations = {}, severity, description, notification = {} } = value; const annotationObj = { ...annotations, description: description }; - const channels = _.get(notification, 'channels') || []; + const channels = _.get(notification, "channels") || []; const matchingLabelsForNotification = await getMatchingLabels(channels); const payload = { grafana_alert: { title: name, condition: condition, - no_data_state: _.get(metadata, 'no_data_state', 'NoData'), - exec_err_state: _.get(metadata, 'exec_err_state', 'Error'), + no_data_state: _.get(metadata, "no_data_state", "NoData"), + exec_err_state: _.get(metadata, "exec_err_state", "Error"), data: metadata, is_paused: false, }, for: interval, annotations: annotationObj, labels: { - 'alertId': id, + "alertId": id, ...labels, ...(severity && { severity }), ...matchingLabelsForNotification diff --git a/api-service/src/services/managers/grafana/alert/index.ts b/api-service/src/services/managers/grafana/alert/index.ts index b2dcd1e2..471a104d 100644 --- a/api-service/src/services/managers/grafana/alert/index.ts +++ b/api-service/src/services/managers/grafana/alert/index.ts @@ -1,5 +1,5 @@ import _ from "lodash"; -import { addGrafanaRule, checkIfGroupNameExists, checkIfRuleExists, createFolderIfNotExists, deleteAlertFolder, deleteAlertRule, getPrometheusDataSource, getQueryExpression, getQueryModel, getSpecificRule, transformRule, updateMetadata, getFilteredAlerts, getRules, groupRulesByCategory } from "./helpers"; +import { addGrafanaRule, checkIfGroupNameExists, checkIfRuleExists, createFolderIfNotExists, deleteAlertFolder, deleteAlertRule, getPrometheusDataSource, getQueryExpression, getQueryModel, getSpecificRule, transformRule, updateMetadata, getRules } from "./helpers"; import { Silence } from "../../../../models/Silence"; import constants from "../../constants"; @@ -27,7 +27,7 @@ const publishAlert = async (payload: Record) => { const getAlerts = async (payload: Record) => { const context = payload?.context || {}; - const alertId = _.get(payload, 'id'); + const alertId = _.get(payload, "id"); const { err, alertData } = await getSpecificRule(payload) .then(alertData => { if (!alertData) throw new Error() @@ -37,21 +37,21 @@ const getAlerts = async (payload: Record) => { const silenceModel = await Silence.findOne({ where: { alert_id: alertId } }); const silenceData = silenceModel?.toJSON(); - let silenceState: Record = { state: '', silenceId: '' }; + const silenceState: Record = { state: "", silenceId: "" }; if (silenceData) { const { end_time } = silenceData; const currentTime = new Date().getTime(); const endTime = new Date(end_time).getTime(); if (currentTime < endTime) { - silenceState.state = 'muted'; - silenceState['endTime'] = endTime; + silenceState.state = "muted"; + silenceState["endTime"] = endTime; } else { - silenceState.state = 'unmuted'; + silenceState.state = "unmuted"; } silenceState.silenceId = silenceData.id; } else { - silenceState.state = 'unmuted'; + silenceState.state = "unmuted"; } return { ...payload, context: { ...context, err }, ...(alertData && { alertData }), silenceState }; @@ -62,8 +62,8 @@ const deleteAlert = async (payload: Record) => { const alertCategory = await checkIfGroupNameExists(category); if (!alertCategory) throw new Error(constants.CATEGORY_NOT_EXIST); - if (_.get(alertCategory, 'rules.length') > 1) { - const filteredRule = _.filter(alertCategory.rules, (rule) => _.get(rule, 'grafana_alert.title') !== name) || []; + if (_.get(alertCategory, "rules.length") > 1) { + const filteredRule = _.filter(alertCategory.rules, (rule) => _.get(rule, "grafana_alert.title") !== name) || []; const filteredGroup = { ...alertCategory, rules: filteredRule }; return addGrafanaRule(filteredGroup); } @@ -83,19 +83,19 @@ const generateAlertPayload = (payload: Record) => { } const filterSystemRulesPredicate = (rule: Record) => { - const labels = _.get(rule, 'labels') || {}; + const labels = _.get(rule, "labels") || {}; const isSystemAlert = _.find(labels, (value, key) => (key === "alertSource" && value === "system-rule-ingestor-job")); if (!isSystemAlert) return true; return false; } -const deleteSystemRules = async (filters: Record) => { +const deleteSystemRules = async () => { const response = await getRules(); - const existingRules = _.cloneDeep(_.get(response, 'data')) as Record[]>; + const existingRules = _.cloneDeep(_.get(response, "data")) as Record[]>; for (const [category, evaluationGroups] of Object.entries(existingRules)) { const evaluationGroup = _.find(evaluationGroups, ["name", category]); if (!evaluationGroup) continue; - const rules = _.get(evaluationGroup, 'rules') || [] + const rules = _.get(evaluationGroup, "rules") || [] const filteredRules = _.filter(rules, filterSystemRulesPredicate); try { if (_.isEmpty(filteredRules)) { diff --git a/api-service/src/services/managers/grafana/index.ts b/api-service/src/services/managers/grafana/index.ts index 30c2b82d..1824c6c5 100644 --- a/api-service/src/services/managers/grafana/index.ts +++ b/api-service/src/services/managers/grafana/index.ts @@ -1,6 +1,6 @@ -import * as alertFunctions from './alert' -import * as notificationFunctions from './notification'; -import * as silenceFunctions from './silences'; +import * as alertFunctions from "./alert" +import * as notificationFunctions from "./notification"; +import * as silenceFunctions from "./silences"; const service = { name: "grafana", ...alertFunctions, ...notificationFunctions, ...silenceFunctions }; export default service \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/email.ts b/api-service/src/services/managers/grafana/notification/channels/email.ts index d4628517..06658016 100644 --- a/api-service/src/services/managers/grafana/notification/channels/email.ts +++ b/api-service/src/services/managers/grafana/notification/channels/email.ts @@ -1,6 +1,6 @@ -import _ from 'lodash'; +import _ from "lodash"; import { IChannelConfig } from "../../../../../types/AlertModels"; -import { grafanaHttpClient } from '../../../../../connections/grafanaConnection'; +import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; const getReceiverObject = ({ name, recipientAddresses, message, multipleAddresses, type }: any) => { return { @@ -30,7 +30,7 @@ const service: IChannelConfig = { generateConfigPayload(payload: Record): Record { const { type, config, name } = payload; const { recipientAddresses, message, subject = "Obsrv Alert", labels = [[`notificationChannel_${name}_${type.toLowerCase()}`, "=", "true"]] } = config; - const multipleAddresses = _.size(_.split(recipientAddresses, ';')) > 1; + const multipleAddresses = _.size(_.split(recipientAddresses, ";")) > 1; return { notificationPolicy: { receiver: name, @@ -42,7 +42,7 @@ const service: IChannelConfig = { testChannel(payload: Record): Promise { const { name, type, config, message = "Test Channel" } = payload; const { recipientAddresses, subject = "Obsrv Alert" } = config; - const multipleAddresses = _.size(_.split(recipientAddresses, ';')) > 1; + const multipleAddresses = _.size(_.split(recipientAddresses, ";")) > 1; const alert = { annotations: { description: message }, labels: {} }; const body = { alert, receivers: [getReceiverObject({ name, type, multipleAddresses, recipientAddresses, subject })] }; return grafanaHttpClient.post("api/alertmanager/grafana/config/api/v1/receivers/test", body); diff --git a/api-service/src/services/managers/grafana/notification/channels/index.ts b/api-service/src/services/managers/grafana/notification/channels/index.ts index e7f5f87f..abc6b45a 100644 --- a/api-service/src/services/managers/grafana/notification/channels/index.ts +++ b/api-service/src/services/managers/grafana/notification/channels/index.ts @@ -1,11 +1,11 @@ -import path from 'path'; -import { scrapModules } from '../../../../../services/fs'; -import { IChannelConfig } from '../../../../../types/AlertModels'; +import path from "path"; +import { scrapModules } from "../../../../../services/fs"; +import { IChannelConfig } from "../../../../../types/AlertModels"; const channels = scrapModules(__dirname, path.basename(__filename)); export const getChannelService = (channelName: string) => { const channel = channels.get(channelName.toLowerCase()); - if (!channel) throw new Error('invalid channel'); + if (!channel) throw new Error("invalid channel"); return channel; } \ No newline at end of file diff --git a/api-service/src/services/managers/grafana/notification/channels/slack.ts b/api-service/src/services/managers/grafana/notification/channels/slack.ts index 2da034fc..983d9879 100644 --- a/api-service/src/services/managers/grafana/notification/channels/slack.ts +++ b/api-service/src/services/managers/grafana/notification/channels/slack.ts @@ -1,7 +1,6 @@ -import axios from 'axios'; -import _ from 'lodash'; -import CONSTANTS from '../../../constants' -import { IChannelConfig } from '../../../../../types/AlertModels'; +import axios from "axios"; +import CONSTANTS from "../../../constants" +import { IChannelConfig } from "../../../../../types/AlertModels"; const generateConfigPayload = (payload: Record): Record => { const { type, config, name } = payload; diff --git a/api-service/src/services/managers/grafana/notification/channels/teams.ts b/api-service/src/services/managers/grafana/notification/channels/teams.ts index 5cc0e689..714f4f19 100644 --- a/api-service/src/services/managers/grafana/notification/channels/teams.ts +++ b/api-service/src/services/managers/grafana/notification/channels/teams.ts @@ -1,7 +1,6 @@ -import axios from 'axios'; -import _ from 'lodash'; -import CONSTANTS from '../../../constants'; -import { IChannelConfig } from '../../../../../types/AlertModels'; +import axios from "axios"; +import CONSTANTS from "../../../constants"; +import { IChannelConfig } from "../../../../../types/AlertModels"; const generateConfigPayload = (payload: Record): Record => { const {type, config, name} = payload; diff --git a/api-service/src/services/managers/grafana/notification/helpers/index.ts b/api-service/src/services/managers/grafana/notification/helpers/index.ts index a90c2ad7..112365b7 100644 --- a/api-service/src/services/managers/grafana/notification/helpers/index.ts +++ b/api-service/src/services/managers/grafana/notification/helpers/index.ts @@ -1,7 +1,7 @@ import _ from "lodash"; import { grafanaHttpClient } from "../../../../../connections/grafanaConnection"; -import { getChannelService } from '../channels'; -import defaultTemplates from '../templates'; +import { getChannelService } from "../channels"; +import defaultTemplates from "../templates"; const generateChannelConfig = (payload: Record) => { const { type } = payload; @@ -18,9 +18,9 @@ const updateAlertManagerConfig = async (payload: Record) => { return grafanaHttpClient.post("/api/alertmanager/grafana/config/api/v1/alerts", payload); }; -const getReceivers = (alertmanager_config: Record) => _.get(alertmanager_config, 'alertmanager_config.receivers') as Array; -const getRoutes = (alertmanager_config: Record) => _.get(alertmanager_config, 'alertmanager_config.route.routes') as Array; -const getTemplates = (alertmanager_config: Record) => _.get(alertmanager_config, 'template_files') as Record; +const getReceivers = (alertmanager_config: Record) => _.get(alertmanager_config, "alertmanager_config.receivers") as Array; +const getRoutes = (alertmanager_config: Record) => _.get(alertmanager_config, "alertmanager_config.route.routes") as Array; +const getTemplates = (alertmanager_config: Record) => _.get(alertmanager_config, "template_files") as Record; const createContactPointsAndNotificationPolicy = async (metadata: Record) => { const { receiver, notificationPolicy } = metadata; @@ -28,9 +28,9 @@ const createContactPointsAndNotificationPolicy = async (metadata: Record, alert const clonedAlertManagerConfig = _.cloneDeep(alertManagerConfig); const existingReceivers = getReceivers(clonedAlertManagerConfig) || []; const existingRoutes = getRoutes(clonedAlertManagerConfig) || []; - _.remove(existingRoutes, route => _.get(route, 'receiver') === name); - _.remove(existingReceivers, receiver => _.get(receiver, 'name') === name); + _.remove(existingRoutes, route => _.get(route, "receiver") === name); + _.remove(existingReceivers, receiver => _.get(receiver, "name") === name); return clonedAlertManagerConfig; }; diff --git a/api-service/src/services/managers/grafana/notification/index.ts b/api-service/src/services/managers/grafana/notification/index.ts index bf7f387b..816e473c 100644 --- a/api-service/src/services/managers/grafana/notification/index.ts +++ b/api-service/src/services/managers/grafana/notification/index.ts @@ -1,5 +1,4 @@ -import _ from "lodash"; -import { getChannelService } from './channels'; +import { getChannelService } from "./channels"; import { createContactPointsAndNotificationPolicy, generateChannelConfig, getAlertManagerConfig, removeReceiverAndNotificationPolicy, updateAlertManagerConfig } from "./helpers"; const updateNotificationChannel = async (payload: Record) => { diff --git a/api-service/src/services/managers/grafana/silences/helpers/index.ts b/api-service/src/services/managers/grafana/silences/helpers/index.ts index 599ab05f..50af380c 100644 --- a/api-service/src/services/managers/grafana/silences/helpers/index.ts +++ b/api-service/src/services/managers/grafana/silences/helpers/index.ts @@ -19,7 +19,7 @@ const disableSilence = (silenceId: string) => { const getCurrentSilenceStatus = async (silenceId: string) => { const response = await getSilence(silenceId); - const currentSilenceStatus = _.get(response, 'data.status.state'); + const currentSilenceStatus = _.get(response, "data.status.state"); return currentSilenceStatus; } diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index e2e57568..84b666d9 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -61,9 +61,9 @@ export const deleteAlertRule = async (payload: Record, hardDelete: export const deleteSystemRules = async (payload: Record) => { - const { rules = [], manager } = payload; + const { manager } = payload; const service = getService(manager); - return service.deleteSystemRules(rules); + return service.deleteSystemRules(); } export const getAlertsMetadata = (payload: Record) => { @@ -125,7 +125,7 @@ export const deleteAlertByDataset = async (payload: Record) => { const { name } = payload; const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) if (!alertRulePayload) throw new Error(constants.ALERTS_NOT_FOUND) - for (let payload of alertRulePayload) { + for (const payload of alertRulePayload) { await deleteAlertRule(payload, true) await retireAlertSilence(_.get(payload, "id") || "") } @@ -140,7 +140,7 @@ export const deleteMetricAliasByDataset = async (payload: Record) = const { name } = payload; const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name } }) if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) - for (let payload of metricAliasPayload) { + for (const payload of metricAliasPayload) { await payload.destroy() } return constants.METRIC_ALIAS_DELETED_SUCCESSFULLY; @@ -173,7 +173,7 @@ export const getAlertMetricsByDataset = async (payload: Record) => export const createAlertsByDataset = async (payload: any) => { try { - for (let alerts of payload) { + for (const alerts of payload) { const alertPayload = _.omit(alerts as any, ["id", "status", "createdAt", "updatedAt", "created_by", "updated_by"]) await Alert.create(alertPayload) } @@ -184,7 +184,7 @@ export const createAlertsByDataset = async (payload: any) => { export const createMetricAliasByDataset = async (payload: any) => { try { - for (let metrics of payload) { + for (const metrics of payload) { const metricsPayload = _.omit(metrics as any, ["id", "createdAt", "updatedAt"]) await Metrics.create(metricsPayload) } @@ -198,7 +198,7 @@ export const publishAlertByDataset = async (payload: Record) => { const { name } = payload; const alertRulePayload = await Alert.findAll({ where: { category: "datasets", "metadata.queryBuilderContext.subComponent": name }, raw: true }) if (!alertRulePayload) throw new Error("Alert rule does not exist") - for (let payload of alertRulePayload) { + for (const payload of alertRulePayload) { await publishAlert(payload) } return constants.ALERTS_PUBLISHED_SUCCESSFULLY; diff --git a/api-service/src/services/managers/prometheus/alert/index.ts b/api-service/src/services/managers/prometheus/alert/index.ts index f2bd3a2f..28f5412f 100644 --- a/api-service/src/services/managers/prometheus/alert/index.ts +++ b/api-service/src/services/managers/prometheus/alert/index.ts @@ -1,17 +1,17 @@ -import CONSTANTS from '../../constants' +import CONSTANTS from "../../constants" -export const publishAlert = async (payload: Record) => { +export const publishAlert = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -export const getAlerts = async (payload: Record) => { +export const getAlerts = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -export const deleteAlert = async (payload: Record) => { +export const deleteAlert = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -export const generateAlertPayload = (payload: Record) => { +export const generateAlertPayload = () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -export const deleteSystemRules= async (payload: Record) => { +export const deleteSystemRules= async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } diff --git a/api-service/src/services/managers/prometheus/index.ts b/api-service/src/services/managers/prometheus/index.ts index c8c2e7df..f8f0da66 100644 --- a/api-service/src/services/managers/prometheus/index.ts +++ b/api-service/src/services/managers/prometheus/index.ts @@ -1,5 +1,5 @@ -import * as alertFunctions from './alert' -import * as notificationFunctions from './notification'; -import * as silenceFunctions from './silences'; +import * as alertFunctions from "./alert" +import * as notificationFunctions from "./notification"; +import * as silenceFunctions from "./silences"; export default { ...alertFunctions, ...notificationFunctions, ...silenceFunctions } \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/notification/index.ts b/api-service/src/services/managers/prometheus/notification/index.ts index 91b663cb..9514d324 100644 --- a/api-service/src/services/managers/prometheus/notification/index.ts +++ b/api-service/src/services/managers/prometheus/notification/index.ts @@ -1,11 +1,11 @@ -import CONSTANTS from '../../constants' +import CONSTANTS from "../../constants" -export const createNotificationChannel = (payload: Record) => { +export const createNotificationChannel = () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } -export const testNotificationChannel = async (payload: Record) => { +export const testNotificationChannel = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } -export const updateNotificationChannel = (payload: Record) => { +export const updateNotificationChannel = () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } \ No newline at end of file diff --git a/api-service/src/services/managers/prometheus/silences/index.ts b/api-service/src/services/managers/prometheus/silences/index.ts index 1c073881..60ed0aef 100644 --- a/api-service/src/services/managers/prometheus/silences/index.ts +++ b/api-service/src/services/managers/prometheus/silences/index.ts @@ -1,18 +1,18 @@ -import CONSTANTS from '../../constants'; +import CONSTANTS from "../../constants"; -const createSilence = async (payload: Record) => { +const createSilence = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } -const getSilenceMetadata = async (payload: Record) => { +const getSilenceMetadata = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED) } -const updateSilence = async (silence: Record, payload: Record) => { +const updateSilence = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } -const deleteSilence = async (payload: Record) => { +const deleteSilence = async () => { throw new Error(CONSTANTS.METHOD_NOT_IMPLEMENTED); } diff --git a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts index a6037fff..487cd77e 100644 --- a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import { TestInputsForDataIngestion } from "./Fixtures"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { Dataset } from "../../../models/Dataset"; import sinon from "sinon"; import { Kafka } from "kafkajs"; @@ -14,17 +14,16 @@ chai.should(); chai.use(chaiHttp); const kafka = new Kafka(connectionConfig.kafka.config); -const producer = kafka.producer(); const apiEndpoint = "/v2/data/in/:datasetId" const resultResponse = [ { - topicName: 'local.test.topic', + topicName: "local.test.topic", partition: 0, errorCode: 0, - baseOffset: '257', - logAppendTime: '-1', - logStartOffset: '0' + baseOffset: "257", + logAppendTime: "-1", + logStartOffset: "0" } ] const kafkaModule = require("../../../connections/kafkaConnection"); @@ -39,7 +38,7 @@ describe("DATA INGEST API", () => { return Promise.resolve({ dataValues: { dataset_config: { - entry_topic: 'local.test.topic', + entry_topic: "local.test.topic", }, extraction_config: { is_batch_event: false, @@ -74,7 +73,7 @@ describe("DATA INGEST API", () => { return Promise.resolve({ dataValues: { dataset_config: { - entry_topic: 'local.test.topic', + entry_topic: "local.test.topic", } } }) @@ -103,7 +102,7 @@ describe("DATA INGEST API", () => { return Promise.resolve({ dataValues: { dataset_config: { - entry_topic: 'local.test.topic', + entry_topic: "local.test.topic", } } }) diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 3a7be3dc..96d53e38 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import nock from "nock"; import { TestQueries } from "./Fixtures"; import { config } from "../../../configs/Config"; -import chaiSpies from 'chai-spies' -import { describe, it } from 'mocha'; +import chaiSpies from "chai-spies" +import { describe, it } from "mocha"; import { Datasource } from "../../../models/Datasource"; chai.use(chaiSpies) chai.should(); @@ -35,7 +35,7 @@ describe("QUERY API TESTS", () => { }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) - .reply(200, ['telemetry-events.1_rollup']) + .reply(200, ["telemetry-events.1_rollup"]) chai .request(app) .post("/v2/data/query/telemetry-events") diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts b/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts index 8f9bd9f7..c11de5f1 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/Fixtures.ts @@ -1,23 +1,23 @@ export const TestQueries = { VALID_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"test","aggregationLevel":"week"},"query":{"queryType":"timeseries","intervals":{"type":"intervals","intervals":["2024-01-31/2024-02-01"]},"granularity":"week","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"},{"type":"filtered","aggregator":{"type":"count","name":"a1"},"filter":{"type":"not","field":{"type":"null","column":"ver"}},"name":"a1"}]}}', - HIGH_LIMIT_NATIVE_QUERY: '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":{"queryType":"timeseries","intervals":{"type":"intervals","intervals":["2024-11-31/2024-12-01"]},"granularity":"day","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"},{"type":"filtered","aggregator":{"type":"count","name":"a1"},"filter":{"type":"not","field":{"type":"null","column":"ver"}},"name":"a1"}],"limit":10000,"threshold":10000}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"test\",\"aggregationLevel\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"2024-01-31/2024-02-01\"]},\"granularity\":\"week\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"},{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a1\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"ver\"}},\"name\":\"a1\"}]}}", + HIGH_LIMIT_NATIVE_QUERY: "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\",\"aggregationLevel\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"2024-11-31/2024-12-01\"]},\"granularity\":\"day\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"},{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a1\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"ver\"}},\"name\":\"a1\"}],\"limit\":10000,\"threshold\":10000}}", WITHOUT_THRESOLD_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events"},"query":{"queryType":"timeBoundary","dimension":"content_status","metric":"count","granularity":"all","intervals":["2020-12-21/2020-12-22"],"aggregations":[]}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\"},\"query\":{\"queryType\":\"timeBoundary\",\"dimension\":\"content_status\",\"metric\":\"count\",\"granularity\":\"all\",\"intervals\":[\"2020-12-21/2020-12-22\"],\"aggregations\":[]}}", VALID_SQL_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"test","aggregationLevel":"week"},"query":"SELECT * FROM \\"test\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"test\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT * FROM \\\"test\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2021-01-21' LIMIT 10\"}", HIGH_LIMIT_SQL_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events"},"querySql":{"query":"SELECT msgid FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-22\' LIMIT 100000"}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\"},\"querySql\":{\"query\":\"SELECT msgid FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2021-01-22' LIMIT 100000\"}}", HIGH_DATE_RANGE_SQL_QUERY: `{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":"SELECT actor_type, content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2022-02-12' LIMIT 10"}`, LIMIT_IS_NAN: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"dataSource":"telemetry-events","aggregationLevel":"week"},"query":"SELECT content_status FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2021-01-01\' AND __time < TIMESTAMP \'2021-01-12\' LIMIT 100"}', - DATASOURCE_NOT_FOUND: '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry"},"query":"SELECT content_status FROM \\"telemetry\\" LIMIT 5"}', - INVALID_DATE_RANGE_NATIVE: '{"id":"api.data.out","ver":"1.0","ts":"1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","table":"week"},"query":{"queryType":"timeseries","intervals":{"type":"intervals","intervals":["2023-01-31/2023-04-01"]},"granularity":"day","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"}]}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"dataSource\":\"telemetry-events\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT content_status FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2021-01-01' AND __time < TIMESTAMP '2021-01-12' LIMIT 100\"}", + DATASOURCE_NOT_FOUND: "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry\"},\"query\":\"SELECT content_status FROM \\\"telemetry\\\" LIMIT 5\"}", + INVALID_DATE_RANGE_NATIVE: "{\"id\":\"api.data.out\",\"ver\":\"1.0\",\"ts\":\"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\",\"table\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"2023-01-31/2023-04-01\"]},\"granularity\":\"day\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"}]}}", INVALID_SQL_QUERY: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry","aggregationLevel":"week"},"query":"SELECT * FROM \\"telemetry\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\' LIMIT 10"}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT * FROM \\\"telemetry\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2021-01-21' LIMIT 10\"}", VALID_SQL_QUERY_WITHOUT_LIMIT: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry","aggregationLevel":"week"},"query":"SELECT * FROM \\"telemetry-events\\" WHERE __time >= TIMESTAMP \'2020-12-31\' AND __time < TIMESTAMP \'2021-01-21\'"}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry\",\"aggregationLevel\":\"week\"},\"query\":\"SELECT * FROM \\\"telemetry-events\\\" WHERE __time >= TIMESTAMP '2020-12-31' AND __time < TIMESTAMP '2021-01-21'\"}", VALID_INTERVAL: - '{"id": "api.data.out","ver": "1.0","ts": "1711966306164","params":{"msgid":"e180ecac-8f41-4f21-9a21-0b3a1a368917"},"context":{"datasetId":"telemetry-events","aggregationLevel":"week"},"query":{"queryType":"timeseries","intervals":"2024-01-31/2024-02-01","granularity":"week","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"msgid"}},"name":"msgid"},{"type":"filtered","aggregator":{"type":"count","name":"a1"},"filter":{"type":"not","field":{"type":"null","column":"ver"}},"name":"a1"}]}}', + "{\"id\": \"api.data.out\",\"ver\": \"1.0\",\"ts\": \"1711966306164\",\"params\":{\"msgid\":\"e180ecac-8f41-4f21-9a21-0b3a1a368917\"},\"context\":{\"datasetId\":\"telemetry-events\",\"aggregationLevel\":\"week\"},\"query\":{\"queryType\":\"timeseries\",\"intervals\":\"2024-01-31/2024-02-01\",\"granularity\":\"week\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"msgid\"}},\"name\":\"msgid\"},{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a1\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"ver\"}},\"name\":\"a1\"}]}}", } \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index 10cd9418..a0e2ace6 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -4,10 +4,9 @@ import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; import { TestInputsForDatasetCreate, DATASET_CREATE_SUCCESS_FIXTURES, DATASET_FAILURE_DUPLICATE_DENORM_FIXTURES } from "./Fixtures"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { sequelize } from "../../../connections/databaseConnection"; -import _ from "lodash"; import { apiId } from "../../../controllers/DatasetCreate/DatasetCreate" import { Dataset } from "../../../models/Dataset"; diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts index b3138c9f..46aaebeb 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -1,5 +1,4 @@ import httpStatus from "http-status" -import _ from "lodash" export const TestInputsForDatasetCreate = { VALID_DATASET: { diff --git a/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts b/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts index 8b55ee75..380ee8ad 100644 --- a/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetList/DatasetList.spec.ts @@ -1,15 +1,14 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { apiId } from "../../../controllers/DatasetList/DatasetList"; import { TestInputsForDatasetList } from "./Fixtures"; import { Dataset } from "../../../models/Dataset"; import { DatasetDraft } from "../../../models/DatasetDraft"; -import { DatasetTransformations } from "../../../models/Transformation"; chai.use(spies); chai.should(); diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 6accc269..8b104ec9 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { apiId } from "../../../controllers/DatasetRead/DatasetRead"; import { TestInputsForDatasetRead } from "./Fixtures"; @@ -25,7 +25,7 @@ describe("DATASET READ API", () => { it("Dataset read success: When minimal fields requested", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ 'name': 'sb-telemetry', 'data_version': 1 }) + return Promise.resolve({ "name": "sb-telemetry", "data_version": 1 }) }) chai .request(app) @@ -36,9 +36,9 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.name.should.be.eq('sb-telemetry') + res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ name: 'sb-telemetry', data_version: 1 })) + result.should.be.eq(JSON.stringify({ name: "sb-telemetry", data_version: 1 })) done(); }); }); @@ -56,8 +56,8 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.type.should.be.eq('event') - res.body.result.status.should.be.eq('Draft') + res.body.result.type.should.be.eq("event") + res.body.result.status.should.be.eq("Draft") const result = JSON.stringify(res.body.result) result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.DRAFT_SCHEMA })) done(); @@ -77,7 +77,7 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.status.should.be.eq('Live') + res.body.result.status.should.be.eq("Live") const result = JSON.stringify(res.body.result) result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA })) done(); @@ -109,7 +109,7 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.name.should.be.eq('sb-telemetry') + res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) done(); @@ -141,7 +141,7 @@ describe("DATASET READ API", () => { res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("SUCCESS") res.body.result.should.be.a("object") - res.body.result.name.should.be.eq('sb-telemetry') + res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) result.should.be.eq(JSON.stringify(TestInputsForDatasetRead.DRAFT_SCHEMA)) done(); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts index f3de6b4f..0016ae5d 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetDelete.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 7c379727..29edc291 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index 8d83d915..388a224d 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index 0e213eb2..c5c32929 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { Dataset } from "../../../models/Dataset"; diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts index da2d4450..fa394a8a 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts index 9f1d5258..e8f614c8 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index 482ed218..cdf02c4b 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -3,7 +3,7 @@ import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts index 964a420f..24ccc831 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts index cea56f57..fe3715d4 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts @@ -3,7 +3,7 @@ import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts index 96da9863..0ae345ff 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts @@ -3,7 +3,7 @@ import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index bec2db18..1818bc79 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -3,7 +3,7 @@ import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts index c69d3de1..e77ab8b0 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; diff --git a/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts b/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts index ee904424..06dccdaf 100644 --- a/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/GenerateSignedURL/Fixtures.ts @@ -55,21 +55,21 @@ export const TestInputsForGenerateURL = { VALID_RESPONSE_FOR_MULTIFILES: [ { "filePath": `container/api-service/user-upload/telemetry.json`, - "fileName": 'telemetry.json', - "preSignedUrl": 'https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC' + "fileName": "telemetry.json", + "preSignedUrl": "https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC" }, { - "filePath": 'container/api-service/user-upload/school-data.json', - "fileName": 'school-data.json', - "preSignedUrl": 'https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/school-data.json?X-Amz-Algorithm=AWS4-HMAC' + "filePath": "container/api-service/user-upload/school-data.json", + "fileName": "school-data.json", + "preSignedUrl": "https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/school-data.json?X-Amz-Algorithm=AWS4-HMAC" } ], VALID_RESPONSE_FOR_SINGLE_FILE: [ { - "filePath": 'container/api-service/user-upload/telemetry.json', - "fileName": 'telemetry.json', - "preSignedUrl": 'https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC' + "filePath": "container/api-service/user-upload/telemetry.json", + "fileName": "telemetry.json", + "preSignedUrl": "https://obsrv-data.s3.ap-south-1.amazonaws.com/container/api-service/user-upload/telemetry.json?X-Amz-Algorithm=AWS4-HMAC" } ] } \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts b/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts index 8d30ddb6..597b3b7c 100644 --- a/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts +++ b/api-service/src/tests/DatasetManagement/GenerateSignedURL/GenerateSignedURL.spec.ts @@ -1,9 +1,9 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai, { expect } from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; import httpStatus from "http-status"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { apiId, code } from "../../../controllers/GenerateSignedURL/GenerateSignedURL"; import { TestInputsForGenerateURL } from "./Fixtures"; diff --git a/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts b/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts index b7f60e27..4bedf75f 100644 --- a/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/CreateTemplate/CreateTemplate.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { createTemplateFixtures } from "./Fixtures" import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.create' +const apiId = "api.query.template.create" const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d"; chai.use(spies); chai.should(); diff --git a/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts b/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts index 4108415b..47377590 100644 --- a/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/DeleteTemplate/DeleteTemplate.spec.ts @@ -1,10 +1,10 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.delete' +const apiId = "api.query.template.delete" chai.use(spies); chai.should(); @@ -20,7 +20,7 @@ describe("DELETE QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "destroy", () => { return Promise.resolve({ dataValues: { - template_id: 'sql1' + template_id: "sql1" } }) }) diff --git a/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts b/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts index e9f843d1..16800dcd 100644 --- a/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts +++ b/api-service/src/tests/QueryTemplates/ListTemplates/ListTemplates.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; import { listTemplateFixtures } from "./Fixtures"; -const apiId = 'api.query.template.list' +const apiId = "api.query.template.list" const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d"; chai.use(spies); chai.should(); diff --git a/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts b/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts index 1f6e129e..6a7078f2 100644 --- a/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/ReadTemplate/ReadTemplate.spec.ts @@ -1,10 +1,10 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.read' +const apiId = "api.query.template.read" chai.use(spies); chai.should(); @@ -20,12 +20,12 @@ describe("READ QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'sql1', - template_name: 'sql1', - query: '"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} LIMIT 1"', - query_type: 'sql', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "sql1", + template_name: "sql1", + query: "\"SELECT * FROM {{DATASET}} WHERE __time BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}} LIMIT 1\"", + query_type: "sql", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04-29T11:29:58.759Z", updated_date: "2024-04-29T11:29:58.759Z" } diff --git a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts index ff384181..8da1818b 100644 --- a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts +++ b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts @@ -1,14 +1,14 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { QueryTemplate } from "../../../models/QueryTemplate"; import { Datasource } from "../../../models/Datasource"; import nock from "nock"; import { config } from "../../../configs/Config"; import { templateQueryApiFixtures } from "./Fixtures"; -const apiId = 'api.query.template.query'; +const apiId = "api.query.template.query"; const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" chai.use(spies); @@ -34,12 +34,12 @@ describe("QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'sql1', - template_name: 'sql1', - query: '"SELECT * FROM {{DATASET}} WHERE \"__time\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}"', - query_type: 'sql', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "sql1", + template_name: "sql1", + query: "\"SELECT * FROM {{DATASET}} WHERE \"__time\" BETWEEN TIMESTAMP {{STARTDATE}} AND TIMESTAMP {{ENDDATE}}\"", + query_type: "sql", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04 - 30T05: 57:04.387Z", updated_date: "2024-04 - 30T05: 57:04.387Z" } @@ -77,12 +77,12 @@ describe("QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'jsontemplate1', - template_name: 'jsontemplate1', - query: '{"queryType":"timeseries","datasetId":"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMIT}}","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"school_id"}},"name":"school_id"}]}', - query_type: 'json', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "jsontemplate1", + template_name: "jsontemplate1", + query: "{\"queryType\":\"timeseries\",\"datasetId\":\"{{DATASET}}\",\"intervals\":\"{{STARTDATE}}/{{ENDDATE}}\",\"limit\":\"{{LIMIT}}\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"school_id\"}},\"name\":\"school_id\"}]}", + query_type: "json", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04-28T23:28:35.868Z", updated_date: "2024-04-28T23:28:35.868Z" } @@ -121,12 +121,12 @@ describe("QUERY TEMPLATE API", () => { chai.spy.on(QueryTemplate, "findOne", () => { return Promise.resolve({ dataValues: { - template_id: 'jsontemplate1', - template_name: 'jsontemplate1', - query: '{"queryType":"timeseries","datasetId"::::"{{DATASET}}","intervals":"{{STARTDATE}}/{{ENDDATE}}","limit":"{{LIMIT}}","aggregations":[{"type":"filtered","aggregator":{"type":"count","name":"a0"},"filter":{"type":"not","field":{"type":"null","column":"school_id"}},"name":"school_id"}]}', - query_type: 'json', - created_by: 'SYSTEM', - updated_by: 'SYSTEM', + template_id: "jsontemplate1", + template_name: "jsontemplate1", + query: "{\"queryType\":\"timeseries\",\"datasetId\"::::\"{{DATASET}}\",\"intervals\":\"{{STARTDATE}}/{{ENDDATE}}\",\"limit\":\"{{LIMIT}}\",\"aggregations\":[{\"type\":\"filtered\",\"aggregator\":{\"type\":\"count\",\"name\":\"a0\"},\"filter\":{\"type\":\"not\",\"field\":{\"type\":\"null\",\"column\":\"school_id\"}},\"name\":\"school_id\"}]}", + query_type: "json", + created_by: "SYSTEM", + updated_by: "SYSTEM", created_date: "2024-04-28T23:28:35.868Z", updated_date: "2024-04-28T23:28:35.868Z" } diff --git a/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts b/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts index 195018c3..af50d77a 100644 --- a/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts +++ b/api-service/src/tests/QueryTemplates/UpdateTemplate/UpdateTemplate.spec.ts @@ -1,11 +1,11 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import { updateTemplateFixtures } from "./Fixtures" import { QueryTemplate } from "../../../models/QueryTemplate"; -const apiId = 'api.query.template.update' +const apiId = "api.query.template.update" const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d"; chai.use(spies); chai.should(); diff --git a/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts b/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts index bd1e1a31..510b8af4 100644 --- a/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts +++ b/api-service/src/tests/QueryWrapper/SqlWrapper/SqlWrapper.spec.ts @@ -1,8 +1,8 @@ -import app from "../../../../app"; +import app from "../../../app"; import chai from "chai"; import chaiHttp from "chai-http"; import spies from "chai-spies"; -import { describe, it } from 'mocha'; +import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForSqlWrapper } from "./Fixtures"; import httpStatus from "http-status"; From 0adc92de744357918003328cfb3d6760324bbf58 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 5 Aug 2024 14:40:59 +0530 Subject: [PATCH 114/235] #OBS-I116: feat: Dataset status transition test cases fix --- .../DatasetLive.spec.ts | 49 ++- .../DatasetReadyToPublish.spec.ts | 10 +- .../DatasetStatusTransition.spec.ts | 13 +- .../DatasetStatusTransition/Fixtures.ts | 307 ++++++++---------- 4 files changed, 177 insertions(+), 202 deletions(-) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 29edc291..bb0c8e51 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -9,6 +9,8 @@ import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { commandHttpService } from "../../../connections/commandServiceConnection"; import { sequelize } from "../../../connections/databaseConnection"; +import { DatasourceDraft } from "../../../models/DatasourceDraft"; +import { Dataset } from "../../../models/Dataset"; chai.use(spies); chai.should(); @@ -24,9 +26,18 @@ describe("DATASET STATUS TRANSITION LIVE", () => { it("Dataset status transition success: When the action is to set dataset live", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "ReadyToPublish" }) + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH) }) - chai.spy.on(commandHttpService, "post", () => { + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { @@ -35,6 +46,9 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(t, "commit", () => { return Promise.resolve({}) }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) chai .request(app) .post("/v2/datasets/status-transition") @@ -47,7 +61,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) res.body.result.message.should.be.eq("Dataset status transition to Live successful") - res.body.result.dataset_id.should.be.eq("telemetry.1") + res.body.result.dataset_id.should.be.eq("telemetry") done(); }); }); @@ -66,7 +80,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }) @@ -74,17 +88,29 @@ describe("DATASET STATUS TRANSITION LIVE", () => { it("Dataset status transition failure: When the command api call to publish dataset fails", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "ReadyToPublish" }) + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH) }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.reject() + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { return Promise.resolve(sequelize.transaction) }) - chai.spy.on(t, "rollback", () => { + chai.spy.on(t, "commit", () => { return Promise.resolve({}) }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.reject() + }) chai .request(app) .post("/v2/datasets/status-transition") @@ -94,26 +120,25 @@ describe("DATASET STATUS TRANSITION LIVE", () => { res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") done(); }); }); it("Dataset status transition failure: When the dataset to publish is in draft state", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Draft" }) + return Promise.resolve({...TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH, status: "Draft"}) }) chai .request(app) .post("/v2/datasets/status-transition") .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); + res.should.have.status(httpStatus.CONFLICT); res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.error.code.should.be.eq("DATASET_LIVE_FAILURE") - res.body.error.message.should.be.eq("Failed to mark dataset Live as it is not in ready to publish state") + res.body.error.message.should.be.eq("Transition failed for dataset: telemetry status:Draft with status transition to Live") done(); }); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index 388a224d..c82e809d 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -47,7 +47,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { res.body.result.should.be.a("object") res.body.params.msgid.should.be.eq(msgid) res.body.result.message.should.be.eq("Dataset status transition to ReadyToPublish successful") - res.body.result.dataset_id.should.be.eq("telemetry.1") + res.body.result.dataset_id.should.be.eq("telemetry") done(); }); }); @@ -66,7 +66,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry.1") + res.body.error.message.should.be.eq("Dataset not found for dataset: telemetry") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); @@ -74,19 +74,19 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { it("Dataset status transition failure: When dataset is already ready to publish", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({dataset_id:"telemetry", status:"ReadyToPublish"}) + return Promise.resolve({ ...TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_READY_TO_PUBLISH, "status": "ReadyToPublish" }) }) chai .request(app) .post("/v2/datasets/status-transition") .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); + res.should.have.status(httpStatus.CONFLICT); res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to mark dataset Ready to publish as it not in draft state") + res.body.error.message.should.be.eq("Transition failed for dataset: dataset-all-fields7 status:ReadyToPublish with status transition to ReadyToPublish") res.body.error.code.should.be.eq("DATASET_READYTOPUBLISH_FAILURE") done(); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts index fa394a8a..7117aa94 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetStatusTransition.spec.ts @@ -22,7 +22,7 @@ describe("DATASET STATUS TRANSITION API", () => { }); it("Dataset status transition failure: Invalid request payload provided", (done) => { - + chai .request(app) .post("/v2/datasets/status-transition") @@ -39,20 +39,21 @@ describe("DATASET STATUS TRANSITION API", () => { }); }); - it("Dataset status transition failure: Connection to the database failed", (done) => { + it("Dataset status transition failure: When the action is performed on v1 apis", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.reject() + return Promise.resolve({ api_version: "v1" }) }) chai .request(app) .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_DELETE) + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); + res.should.have.status(httpStatus.BAD_REQUEST); res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") + res.body.error.code.should.be.eq("DATASET_API_VERSION_MISMATCH") + res.body.error.message.should.be.eq("Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset") done(); }); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index 7deb12f7..db14c924 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -1,187 +1,136 @@ export const TestInputsForDatasetStatusTransition = { - VALID_SCHEMA_FOR_DELETE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "Delete" - } + VALID_SCHEMA_FOR_DELETE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - VALID_SCHEMA_FOR_LIVE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "Live" - } + "request": { + "dataset_id": "telemetry.1", + "status": "Delete" + } + }, + VALID_SCHEMA_FOR_LIVE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - VALID_SCHEMA_FOR_RETIRE: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry", - "status": "Retire" - } + "request": { + "dataset_id": "telemetry", + "status": "Live" + } + }, + VALID_SCHEMA_FOR_RETIRE: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - INVALID_SCHEMA: { - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "" - } + "request": { + "dataset_id": "telemetry", + "status": "Retire" + } + }, + INVALID_SCHEMA: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" }, - VALID_REQUEST_FOR_READY_FOR_PUBLISH:{ - "id": "api.datasets.status-transition", - "ver": "v2", - "ts": "2024-04-19T12:58:47+05:30", - "params": { - "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" - }, - "request": { - "dataset_id": "telemetry.1", - "status": "ReadyToPublish" + "request": { + "dataset_id": "telemetry.1", + "status": "" + } + }, + VALID_REQUEST_FOR_READY_FOR_PUBLISH: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "telemetry", + "status": "ReadyToPublish" + } + }, + VALID_SCHEMA_FOR_READY_TO_PUBLISH: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "event", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": true, "cache_enabled": false }, "keys_config": { "data_key": "eid", "partition_key": "eid", "timestamp_key": "obsrv_meta.syncts" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": [] }, + "tags": ["tag1"], + "status": "Draft", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-07-24 19:12:13.021", + "updated_date": "2024-07-25 06:12:38.412", + "version_key": "1721887933020", + "api_version": "v2", + "transformations_config": [{ "field_key": "email", "transformation_function": { "type": "mask", "expr": "mid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], + "connectors_config": [{ "id": "91898e828u82882u8", "connector_id": "kafka", "connector_config": "AR/hz8iBxRyc9s0ohXa3+id+7GoWtjVjNWvurFFgV1Ocw2kgc+XVbnfXX26zkP3+rQ49gio0JzwsFzOK61TtXLx968IKol5eGfaEHF68O5faoxxjKBsyvhPaRQ91DKKi", "version": "v1" }], + "sample_data": {}, + "entry_topic": "local.ingest" + }, + INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { + "dataset_id": "telemetry", + "type": "", + "name": "sb-telemetry", + "id": "telemetry.1", + "status": "Draft", + "version_key": "1789887878", + "validation_config": { + "validate": true, + "mode": "Strict" + }, + "router_config": { + "topic": "test" + }, + "denorm_config": { + "redis_db_host": "local", + "redis_db_port": 5432, + "denorm_fields": [ + { + "denorm_key": "actor.id", + "denorm_out_field": "userdata", + "dataset_name": "name", + "dataset_id": "name" + }, + { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_name": "name", + "dataset_id": "name" } + ] }, - VALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "dataset", - "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "extraction_config": { - "is_batch_event": true, - "extraction_key": "events", - "batch_id": "id", - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "id", - "dedup_period": 3783 - } - }, - "dedup_config": { - "drop_duplicates": true, - "dedup_key": "mid", - "dedup_period": 3783 - }, - "data_schema": { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "properties": { - "ets": { - "type": "string" - }, - "ver": { - "type": "string" - }, - "required": [ - "eid" - ] - }, - "additionalProperties": true - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" - ] - }, - INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "", - "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" - ] - } + "dataset_config": { + "data_key": "mid", + "timestamp_key": "ets", + "entry_topic": "topic", + "redis_db_host": "local", + "redis_db_port": 5432, + "redis_db": 0, + "index_data": true + }, + "client_state": {}, + "tags": [ + "tag1", + "tag2" + ] + }, + DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } } \ No newline at end of file From b537062b571462513c8f4b8584a6b1e3365c81b4 Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Mon, 5 Aug 2024 17:10:33 +0530 Subject: [PATCH 115/235] #OBS-I1 updated the routes --- api-service/src/routes/Router.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 6ff7eb48..3f2d623a 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -46,8 +46,8 @@ router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); -router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), DatasetCopy); From 91ca4d85cb908fcac85e54ab3cb60a93725ac1df Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 6 Aug 2024 08:04:02 +0530 Subject: [PATCH 116/235] #OBS-I116: feat: Dataset migratio method fix --- api-service/src/services/DatasetService.ts | 38 ++++------------------ 1 file changed, 7 insertions(+), 31 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 9f61ed3f..25b120f2 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -94,35 +94,8 @@ class DatasetService { migrateDraftDataset = async (datasetId: string, dataset: Record): Promise => { const dataset_id = _.get(dataset, "id") - const dataset_config: any = _.get(dataset, "dataset_config"); const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); const transaction = await sequelize.transaction(); - - draftDataset["dataset_config"] = { - indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, - keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, - cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} - } - const transformations = await this.getDraftTransformations(dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); - draftDataset["transformations_config"] = _.map(transformations, (config) => { - return { - field_key: _.get(config, ["field_key"]), - transformation_function: _.get(config, ["transformation_function"]), - mode: _.get(config, ["mode"]), - datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) - } - }) - const connectors = await this.getDraftConnectors(dataset_id, ["id", "connector_type", "connector_config"]); - draftDataset["connectors_config"] = _.map(connectors, (config) => { - return { - id: _.get(config, ["id"]), - connector_id: _.get(config, ["connector_type"]), - connector_config: _.get(config, ["connector_config"]), - version: "v1" - } - }) - draftDataset["sample_data"] = dataset_config?.mergedEvent try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); @@ -173,6 +146,7 @@ class DatasetService { } }) draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) + draftDataset["sample_data"] = dataset_config?.mergedEvent return draftDataset; } @@ -212,10 +186,12 @@ class DatasetService { draftDataset["transformations_config"] = _.map(transformations, (config) => { return { field_key: _.get(config, "field_key"), - transformation_function: _.get(config, "transformation_function"), - mode: _.get(config, "mode"), - datatype: _.get(config, "metadata._transformedFieldDataType") || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + transformation_function: { + ..._.get(config, ["transformation_function"]), + datatype: _.get(config, "metadata._transformedFieldDataType") || "string", + category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + }, + mode: _.get(config, "mode") } }) draftDataset["api_version"] = "v2" From 48db330d4a569f86e75c159d5a422257b253fc15 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Tue, 6 Aug 2024 18:36:50 +0530 Subject: [PATCH 117/235] #OBS-I108: feat: helm modifications for flink connectors --- command-service/helm-charts/flink/Chart.lock | 18 + command-service/helm-charts/flink/Chart.yaml | 7 + .../helm-charts/flink/charts/.helmignore | 23 + .../flink/charts/common/Chart.yaml | 12 + .../charts/common/templates/_affinities.tpl | 139 ++++ .../charts/common/templates/_capabilities.tpl | 229 +++++++ .../charts/common/templates/_configs.tpl | 0 .../flink/charts/common/templates/_errors.tpl | 28 + .../flink/charts/common/templates/_images.tpl | 117 ++++ .../charts/common/templates/_ingress.tpl | 73 +++ .../flink/charts/common/templates/_labels.tpl | 46 ++ .../flink/charts/common/templates/_names.tpl | 71 ++ .../charts/common/templates/_secrets.tpl | 172 +++++ .../charts/common/templates/_storage.tpl | 28 + .../charts/common/templates/_tplvalues.tpl | 38 ++ .../flink/charts/common/templates/_utils.tpl | 77 +++ .../charts/common/templates/_variables.tpl | 41 ++ .../charts/common/templates/_warnings.tpl | 19 + .../templates/validations/_cassandra.tpl | 77 +++ .../common/templates/validations/_mariadb.tpl | 108 ++++ .../common/templates/validations/_mongodb.tpl | 113 ++++ .../common/templates/validations/_mysql.tpl | 108 ++++ .../templates/validations/_postgresql.tpl | 134 ++++ .../common/templates/validations/_redis.tpl | 81 +++ .../templates/validations/_validations.tpl | 51 ++ .../flink/charts/common/values.yaml | 82 +++ .../helm-charts/flink/templates/NOTES.txt | 0 .../flink/templates/_base_serviceAccount.tpl | 4 + .../helm-charts/flink/templates/_helpers.tpl | 0 .../flink/templates/_image_flink.tpl | 15 + .../flink/templates/_namespace.tpl | 13 + .../flink/templates/configmap.yaml | 30 + .../flink/templates/deployment.yaml | 220 +++++++ .../helm-charts/flink/templates/hpa.yaml | 20 + .../helm-charts/flink/templates/ingress.yaml | 24 + .../helm-charts/flink/templates/service.yaml | 42 ++ .../flink/templates/serviceaccount.yaml | 11 + .../flink/templates/servicemonitor.yaml | 31 + command-service/helm-charts/flink/values.yaml | 611 ++++++++++++++++++ 39 files changed, 2913 insertions(+) create mode 100644 command-service/helm-charts/flink/Chart.lock create mode 100644 command-service/helm-charts/flink/Chart.yaml create mode 100644 command-service/helm-charts/flink/charts/.helmignore create mode 100644 command-service/helm-charts/flink/charts/common/Chart.yaml create mode 100644 command-service/helm-charts/flink/charts/common/templates/_affinities.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_capabilities.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_configs.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_errors.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_images.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_ingress.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_labels.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_names.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_secrets.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_storage.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_tplvalues.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_utils.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_variables.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/_warnings.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/validations/_cassandra.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/validations/_mariadb.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/validations/_mongodb.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/validations/_mysql.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/validations/_postgresql.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/validations/_redis.tpl create mode 100644 command-service/helm-charts/flink/charts/common/templates/validations/_validations.tpl create mode 100644 command-service/helm-charts/flink/charts/common/values.yaml create mode 100644 command-service/helm-charts/flink/templates/NOTES.txt create mode 100644 command-service/helm-charts/flink/templates/_base_serviceAccount.tpl create mode 100644 command-service/helm-charts/flink/templates/_helpers.tpl create mode 100644 command-service/helm-charts/flink/templates/_image_flink.tpl create mode 100644 command-service/helm-charts/flink/templates/_namespace.tpl create mode 100644 command-service/helm-charts/flink/templates/configmap.yaml create mode 100644 command-service/helm-charts/flink/templates/deployment.yaml create mode 100644 command-service/helm-charts/flink/templates/hpa.yaml create mode 100644 command-service/helm-charts/flink/templates/ingress.yaml create mode 100644 command-service/helm-charts/flink/templates/service.yaml create mode 100644 command-service/helm-charts/flink/templates/serviceaccount.yaml create mode 100644 command-service/helm-charts/flink/templates/servicemonitor.yaml create mode 100644 command-service/helm-charts/flink/values.yaml diff --git a/command-service/helm-charts/flink/Chart.lock b/command-service/helm-charts/flink/Chart.lock new file mode 100644 index 00000000..fc1e9b7d --- /dev/null +++ b/command-service/helm-charts/flink/Chart.lock @@ -0,0 +1,18 @@ +dependencies: +- name: common + repository: https://nimbushubin.github.io/helmcharts + version: 0.1.0 +- name: redis + repository: https://charts.bitnami.com/bitnami + version: 18.1.1 +- name: kafka + repository: https://charts.bitnami.com/bitnami + version: 20.0.2 +- name: cassandra + repository: https://charts.bitnami.com/bitnami + version: 10.1.0 +- name: postgresql + repository: https://charts.bitnami.com/bitnami + version: 12.2.7 +digest: sha256:7ebab454bd79d34678de3d11bcc4b10c25f0821d693e7a7399e4c8bebe742313 +generated: "2023-12-12T13:15:34.600338+05:30" diff --git a/command-service/helm-charts/flink/Chart.yaml b/command-service/helm-charts/flink/Chart.yaml new file mode 100644 index 00000000..7b2891b5 --- /dev/null +++ b/command-service/helm-charts/flink/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +appVersion: 1.0.0 +description: A production-ready Helm chart base template +maintainers: + - name: NimbusHub.in +name: flink +version: 0.1.0 diff --git a/command-service/helm-charts/flink/charts/.helmignore b/command-service/helm-charts/flink/charts/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/command-service/helm-charts/flink/charts/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/command-service/helm-charts/flink/charts/common/Chart.yaml b/command-service/helm-charts/flink/charts/common/Chart.yaml new file mode 100644 index 00000000..cc1e9676 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +description: Library chart for Sunbird +keywords: +- common +- helper +- template +- function +maintainers: +- name: NimbusHub.in +name: common +type: library +version: 0.1.0 diff --git a/command-service/helm-charts/flink/charts/common/templates/_affinities.tpl b/command-service/helm-charts/flink/charts/common/templates/_affinities.tpl new file mode 100644 index 00000000..e85b1df4 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_affinities.tpl @@ -0,0 +1,139 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return a soft nodeAffinity definition +{{ include "common.affinities.nodes.soft" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.soft" -}} +preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} + weight: 1 +{{- end -}} + +{{/* +Return a hard nodeAffinity definition +{{ include "common.affinities.nodes.hard" (dict "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes.hard" -}} +requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: {{ .key }} + operator: In + values: + {{- range .values }} + - {{ . | quote }} + {{- end }} +{{- end -}} + +{{/* +Return a nodeAffinity definition +{{ include "common.affinities.nodes" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.nodes" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.nodes.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.nodes.hard" . -}} + {{- end -}} +{{- end -}} + +{{/* +Return a topologyKey definition +{{ include "common.affinities.topologyKey" (dict "topologyKey" "BAR") -}} +*/}} +{{- define "common.affinities.topologyKey" -}} +{{ .topologyKey | default "kubernetes.io/hostname" -}} +{{- end -}} + +{{/* +Return a soft podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.soft" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.soft" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: 1 + {{- range $extraPodAffinityTerms }} + - podAffinityTerm: + labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 10 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + weight: {{ .weight | default 1 -}} + {{- end -}} +{{- end -}} + +{{/* +Return a hard podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods.hard" (dict "component" "FOO" "customLabels" .Values.podLabels "extraMatchLabels" .Values.extraMatchLabels "topologyKey" "BAR" "extraPodAffinityTerms" .Values.extraPodAffinityTerms "context" $) -}} +*/}} +{{- define "common.affinities.pods.hard" -}} +{{- $component := default "" .component -}} +{{- $customLabels := default (dict) .customLabels -}} +{{- $extraMatchLabels := default (dict) .extraMatchLabels -}} +{{- $extraPodAffinityTerms := default (list) .extraPodAffinityTerms -}} +requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" .context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := $extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- range $extraPodAffinityTerms }} + - labelSelector: + matchLabels: {{- (include "common.labels.matchLabels" ( dict "customLabels" $customLabels "context" $.context )) | nindent 8 }} + {{- if not (empty $component) }} + {{ printf "app.kubernetes.io/component: %s" $component }} + {{- end }} + {{- range $key, $value := .extraMatchLabels }} + {{ $key }}: {{ $value | quote }} + {{- end }} + topologyKey: {{ include "common.affinities.topologyKey" (dict "topologyKey" .topologyKey) }} + {{- end -}} +{{- end -}} + +{{/* +Return a podAffinity/podAntiAffinity definition +{{ include "common.affinities.pods" (dict "type" "soft" "key" "FOO" "values" (list "BAR" "BAZ")) -}} +*/}} +{{- define "common.affinities.pods" -}} + {{- if eq .type "soft" }} + {{- include "common.affinities.pods.soft" . -}} + {{- else if eq .type "hard" }} + {{- include "common.affinities.pods.hard" . -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_capabilities.tpl b/command-service/helm-charts/flink/charts/common/templates/_capabilities.tpl new file mode 100644 index 00000000..b1257397 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_capabilities.tpl @@ -0,0 +1,229 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Return the target Kubernetes version +*/}} +{{- define "common.capabilities.kubeVersion" -}} +{{- if .Values.global }} + {{- if .Values.global.kubeVersion }} + {{- .Values.global.kubeVersion -}} + {{- else }} + {{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} + {{- end -}} +{{- else }} +{{- default .Capabilities.KubeVersion.Version .Values.kubeVersion -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for poddisruptionbudget. +*/}} +{{- define "common.capabilities.policy.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "policy/v1beta1" -}} +{{- else -}} +{{- print "policy/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "common.capabilities.networkPolicy.apiVersion" -}} +{{- if semverCompare "<1.7-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for cronjob. +*/}} +{{- define "common.capabilities.cronjob.apiVersion" -}} +{{- if semverCompare "<1.21-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "batch/v1beta1" -}} +{{- else -}} +{{- print "batch/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for daemonset. +*/}} +{{- define "common.capabilities.daemonset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "common.capabilities.deployment.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "common.capabilities.statefulset.apiVersion" -}} +{{- if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apps/v1beta1" -}} +{{- else -}} +{{- print "apps/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "common.capabilities.ingress.apiVersion" -}} +{{- if .Values.ingress -}} +{{- if .Values.ingress.apiVersion -}} +{{- .Values.ingress.apiVersion -}} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end }} +{{- else if semverCompare "<1.14-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "extensions/v1beta1" -}} +{{- else if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "networking.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "networking.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for RBAC resources. +*/}} +{{- define "common.capabilities.rbac.apiVersion" -}} +{{- if semverCompare "<1.17-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "rbac.authorization.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "rbac.authorization.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for CRDs. +*/}} +{{- define "common.capabilities.crd.apiVersion" -}} +{{- if semverCompare "<1.19-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiextensions.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiextensions.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for APIService. +*/}} +{{- define "common.capabilities.apiService.apiVersion" -}} +{{- if semverCompare "<1.10-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiregistration.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiregistration.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Horizontal Pod Autoscaler. +*/}} +{{- define "common.capabilities.hpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for Vertical Pod Autoscaler. +*/}} +{{- define "common.capabilities.vpa.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .context) -}} +{{- if .beta2 -}} +{{- print "autoscaling/v2beta2" -}} +{{- else -}} +{{- print "autoscaling/v2beta1" -}} +{{- end -}} +{{- else -}} +{{- print "autoscaling/v2" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if PodSecurityPolicy is supported +*/}} +{{- define "common.capabilities.psp.supported" -}} +{{- if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if AdmissionConfiguration is supported +*/}} +{{- define "common.capabilities.admisionConfiguration.supported" -}} +{{- if semverCompare ">=1.23-0" (include "common.capabilities.kubeVersion" .) -}} + {{- true -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for AdmissionConfiguration. +*/}} +{{- define "common.capabilities.admisionConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "apiserver.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "apiserver.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for PodSecurityConfiguration. +*/}} +{{- define "common.capabilities.podSecurityConfiguration.apiVersion" -}} +{{- if semverCompare "<1.23-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1alpha1" -}} +{{- else if semverCompare "<1.25-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "pod-security.admission.config.k8s.io/v1beta1" -}} +{{- else -}} +{{- print "pod-security.admission.config.k8s.io/v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the used Helm version is 3.3+. +A way to check the used Helm version was not introduced until version 3.3.0 with .Capabilities.HelmVersion, which contains an additional "{}}" structure. +This check is introduced as a regexMatch instead of {{ if .Capabilities.HelmVersion }} because checking for the key HelmVersion in <3.3 results in a "interface not found" error. +**To be removed when the catalog's minimun Helm version is 3.3** +*/}} +{{- define "common.capabilities.supportsHelmVersion" -}} +{{- if regexMatch "{(v[0-9])*[^}]*}}$" (.Capabilities | toString ) }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_configs.tpl b/command-service/helm-charts/flink/charts/common/templates/_configs.tpl new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/flink/charts/common/templates/_errors.tpl b/command-service/helm-charts/flink/charts/common/templates/_errors.tpl new file mode 100644 index 00000000..07ded6f6 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_errors.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Through error when upgrading using empty passwords values that must not be empty. + +Usage: +{{- $validationError00 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password00" "secret" "secretName" "field" "password-00") -}} +{{- $validationError01 := include "common.validations.values.single.empty" (dict "valueKey" "path.to.password01" "secret" "secretName" "field" "password-01") -}} +{{ include "common.errors.upgrade.passwords.empty" (dict "validationErrors" (list $validationError00 $validationError01) "context" $) }} + +Required password params: + - validationErrors - String - Required. List of validation strings to be return, if it is empty it won't throw error. + - context - Context - Required. Parent context. +*/}} +{{- define "common.errors.upgrade.passwords.empty" -}} + {{- $validationErrors := join "" .validationErrors -}} + {{- if and $validationErrors .context.Release.IsUpgrade -}} + {{- $errorString := "\nPASSWORDS ERROR: You must provide your current passwords when upgrading the release." -}} + {{- $errorString = print $errorString "\n Note that even after reinstallation, old credentials may be needed as they may be kept in persistent volume claims." -}} + {{- $errorString = print $errorString "\n Further information can be obtained at https://docs.bitnami.com/general/how-to/troubleshoot-helm-chart-issues/#credential-errors-while-upgrading-chart-releases" -}} + {{- $errorString = print $errorString "\n%s" -}} + {{- printf $errorString $validationErrors | fail -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_images.tpl b/command-service/helm-charts/flink/charts/common/templates/_images.tpl new file mode 100644 index 00000000..1bcb779d --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_images.tpl @@ -0,0 +1,117 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper image name +{{ include "common.images.image" ( dict "imageRoot" .Values.path.to.the.image "global" .Values.global ) }} +*/}} +{{- define "common.images.image" -}} +{{- $registryName := .imageRoot.registry -}} +{{- $repositoryName := .imageRoot.repository -}} +{{- $separator := ":" -}} +{{- $termination := .imageRoot.tag | toString -}} +{{- if .global }} + {{- if .global.imageRegistry }} + {{- $registryName = .global.imageRegistry -}} + {{- end -}} +{{- end -}} +{{- if .imageRoot.digest }} + {{- $separator = "@" -}} + {{- $termination = .imageRoot.digest | toString -}} +{{- end -}} +{{- if $registryName }} + {{- printf "%s/%s%s%s" $registryName $repositoryName $separator $termination -}} +{{- else -}} + {{- printf "%s%s%s" $repositoryName $separator $termination -}} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names (deprecated: use common.images.renderPullSecrets instead) +{{ include "common.images.pullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "global" .Values.global) }} +*/}} +{{- define "common.images.pullSecrets" -}} + {{- $pullSecrets := list }} + + {{- if .global }} + {{- range .global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end }} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets .name -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets . -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names evaluating values as templates +{{ include "common.images.renderPullSecrets" ( dict "images" (list .Values.path.to.the.image1, .Values.path.to.the.image2) "context" $) }} +*/}} +{{- define "common.images.renderPullSecrets" -}} + {{- $pullSecrets := list }} + {{- $context := .context }} + + {{- if $context.Values.global }} + {{- range $context.Values.global.imagePullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- range .images -}} + {{- range .pullSecrets -}} + {{- if kindIs "map" . -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" .name "context" $context)) -}} + {{- else -}} + {{- $pullSecrets = append $pullSecrets (include "common.tplvalues.render" (dict "value" . "context" $context)) -}} + {{- end -}} + {{- end -}} + {{- end -}} + + {{- if (not (empty $pullSecrets)) }} +imagePullSecrets: + {{- range $pullSecrets | uniq }} + - name: {{ . }} + {{- end }} + {{- end }} +{{- end -}} + +{{/* +Return the proper image version (ingores image revision/prerelease info & fallbacks to chart appVersion) +{{ include "common.images.version" ( dict "imageRoot" .Values.path.to.the.image "chart" .Chart ) }} +*/}} +{{- define "common.images.version" -}} +{{- $imageTag := .imageRoot.tag | toString -}} +{{/* regexp from https://github.com/Masterminds/semver/blob/23f51de38a0866c5ef0bfc42b3f735c73107b700/version.go#L41-L44 */}} +{{- if regexMatch `^([0-9]+)(\.[0-9]+)?(\.[0-9]+)?(-([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?(\+([0-9A-Za-z\-]+(\.[0-9A-Za-z\-]+)*))?$` $imageTag -}} + {{- $version := semver $imageTag -}} + {{- printf "%d.%d.%d" $version.Major $version.Minor $version.Patch -}} +{{- else -}} + {{- print .chart.AppVersion -}} +{{- end -}} +{{- end -}} + diff --git a/command-service/helm-charts/flink/charts/common/templates/_ingress.tpl b/command-service/helm-charts/flink/charts/common/templates/_ingress.tpl new file mode 100644 index 00000000..efa5b85c --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_ingress.tpl @@ -0,0 +1,73 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Generate backend entry that is compatible with all Kubernetes API versions. + +Usage: +{{ include "common.ingress.backend" (dict "serviceName" "backendName" "servicePort" "backendPort" "context" $) }} + +Params: + - serviceName - String. Name of an existing service backend + - servicePort - String/Int. Port name (or number) of the service. It will be translated to different yaml depending if it is a string or an integer. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.ingress.backend" -}} +{{- $apiVersion := (include "common.capabilities.ingress.apiVersion" .context) -}} +{{- if or (eq $apiVersion "extensions/v1beta1") (eq $apiVersion "networking.k8s.io/v1beta1") -}} +serviceName: {{ .serviceName }} +servicePort: {{ .servicePort }} +{{- else -}} +service: + name: {{ .serviceName }} + port: + {{- if typeIs "string" .servicePort }} + name: {{ .servicePort }} + {{- else if or (typeIs "int" .servicePort) (typeIs "float64" .servicePort) }} + number: {{ .servicePort | int }} + {{- end }} +{{- end -}} +{{- end -}} + +{{/* +Print "true" if the API pathType field is supported +Usage: +{{ include "common.ingress.supportsPathType" . }} +*/}} +{{- define "common.ingress.supportsPathType" -}} +{{- if (semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .)) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Returns true if the ingressClassname field is supported +Usage: +{{ include "common.ingress.supportsIngressClassname" . }} +*/}} +{{- define "common.ingress.supportsIngressClassname" -}} +{{- if semverCompare "<1.18-0" (include "common.capabilities.kubeVersion" .) -}} +{{- print "false" -}} +{{- else -}} +{{- print "true" -}} +{{- end -}} +{{- end -}} + +{{/* +Return true if cert-manager required annotations for TLS signed +certificates are set in the Ingress annotations +Ref: https://cert-manager.io/docs/usage/ingress/#supported-annotations +Usage: +{{ include "common.ingress.certManagerRequest" ( dict "annotations" .Values.path.to.the.ingress.annotations ) }} +*/}} +{{- define "common.ingress.certManagerRequest" -}} +{{ if or (hasKey .annotations "cert-manager.io/cluster-issuer") (hasKey .annotations "cert-manager.io/issuer") (hasKey .annotations "kubernetes.io/tls-acme") }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_labels.tpl b/command-service/helm-charts/flink/charts/common/templates/_labels.tpl new file mode 100644 index 00000000..d90a6cdc --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_labels.tpl @@ -0,0 +1,46 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} + +{{/* +Kubernetes standard labels +{{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) -}} +*/}} +{{- define "common.labels.standard" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{- $default := dict "app.kubernetes.io/name" (include "common.names.name" .context) "helm.sh/chart" (include "common.names.chart" .context) "app.kubernetes.io/instance" .context.Release.Name "app.kubernetes.io/managed-by" .context.Release.Service -}} +{{- with .context.Chart.AppVersion -}} +{{- $_ := set $default "app.kubernetes.io/version" . -}} +{{- end -}} +{{ template "common.tplvalues.merge" (dict "values" (list .customLabels $default) "context" .context) }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +helm.sh/chart: {{ include "common.names.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Labels used on immutable fields such as deploy.spec.selector.matchLabels or svc.spec.selector +{{ include "common.labels.matchLabels" (dict "customLabels" .Values.podLabels "context" $) -}} + +We don't want to loop over custom labels appending them to the selector +since it's very likely that it will break deployments, services, etc. +However, it's important to overwrite the standard labels if the user +overwrote them on metadata.labels fields. +*/}} +{{- define "common.labels.matchLabels" -}} +{{- if and (hasKey . "customLabels") (hasKey . "context") -}} +{{ merge (pick (include "common.tplvalues.render" (dict "value" .customLabels "context" .context) | fromYaml) "app.kubernetes.io/name" "app.kubernetes.io/instance") (dict "app.kubernetes.io/name" (include "common.names.name" .context) "app.kubernetes.io/instance" .context.Release.Name ) | toYaml }} +{{- else -}} +app.kubernetes.io/name: {{ include "common.names.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_names.tpl b/command-service/helm-charts/flink/charts/common/templates/_names.tpl new file mode 100644 index 00000000..a222924f --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_names.tpl @@ -0,0 +1,71 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "common.names.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "common.names.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "common.names.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create a default fully qualified dependency name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +Usage: +{{ include "common.names.dependency.fullname" (dict "chartName" "dependency-chart-name" "chartValues" .Values.dependency-chart "context" $) }} +*/}} +{{- define "common.names.dependency.fullname" -}} +{{- if .chartValues.fullnameOverride -}} +{{- .chartValues.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .chartName .chartValues.nameOverride -}} +{{- if contains $name .context.Release.Name -}} +{{- .context.Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .context.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts. +*/}} +{{- define "common.names.namespace" -}} +{{- default .Release.Namespace .Values.namespaceOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a fully qualified app name adding the installation's namespace. +*/}} +{{- define "common.names.fullname.namespace" -}} +{{- printf "%s-%s" (include "common.names.fullname" .) (include "common.names.namespace" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_secrets.tpl b/command-service/helm-charts/flink/charts/common/templates/_secrets.tpl new file mode 100644 index 00000000..a193c46b --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_secrets.tpl @@ -0,0 +1,172 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Generate secret name. + +Usage: +{{ include "common.secrets.name" (dict "existingSecret" .Values.path.to.the.existingSecret "defaultNameSuffix" "mySuffix" "context" $) }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - defaultNameSuffix - String - Optional. It is used only if we have several secrets in the same deployment. + - context - Dict - Required. The context for the template evaluation. +*/}} +{{- define "common.secrets.name" -}} +{{- $name := (include "common.names.fullname" .context) -}} + +{{- if .defaultNameSuffix -}} +{{- $name = printf "%s-%s" $name .defaultNameSuffix | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- with .existingSecret -}} +{{- if not (typeIs "string" .) -}} +{{- with .name -}} +{{- $name = . -}} +{{- end -}} +{{- else -}} +{{- $name = . -}} +{{- end -}} +{{- end -}} + +{{- printf "%s" $name -}} +{{- end -}} + +{{/* +Generate secret key. + +Usage: +{{ include "common.secrets.key" (dict "existingSecret" .Values.path.to.the.existingSecret "key" "keyName") }} + +Params: + - existingSecret - ExistingSecret/String - Optional. The path to the existing secrets in the values.yaml given by the user + to be used instead of the default one. Allows for it to be of type String (just the secret name) for backwards compatibility. + +info: https://github.com/bitnami/charts/tree/main/bitnami/common#existingsecret + - key - String - Required. Name of the key in the secret. +*/}} +{{- define "common.secrets.key" -}} +{{- $key := .key -}} + +{{- if .existingSecret -}} + {{- if not (typeIs "string" .existingSecret) -}} + {{- if .existingSecret.keyMapping -}} + {{- $key = index .existingSecret.keyMapping $.key -}} + {{- end -}} + {{- end }} +{{- end -}} + +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Generate secret password or retrieve one if already created. + +Usage: +{{ include "common.secrets.passwords.manage" (dict "secret" "secret-name" "key" "keyName" "providedValues" (list "path.to.password1" "path.to.password2") "length" 10 "strong" false "chartName" "chartName" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - providedValues - List - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - length - int - Optional - Length of the generated random password. + - strong - Boolean - Optional - Whether to add symbols to the generated random password. + - chartName - String - Optional - Name of the chart used when said chart is deployed as a subchart. + - context - Context - Required - Parent context. + - failOnNew - Boolean - Optional - Default to true. If set to false, skip errors adding new keys to existing secrets. +The order in which this function returns a secret password: + 1. Already existing 'Secret' resource + (If a 'Secret' resource is found under the name provided to the 'secret' parameter to this function and that 'Secret' resource contains a key with the name passed as the 'key' parameter to this function then the value of this existing secret password will be returned) + 2. Password provided via the values.yaml + (If one of the keys passed to the 'providedValues' parameter to this function is a valid path to a key in the values.yaml and has a value, the value of the first key with a value will be returned) + 3. Randomly generated secret password + (A new random secret password with the length specified in the 'length' parameter will be generated and returned) + +*/}} +{{- define "common.secrets.passwords.manage" -}} + +{{- $password := "" }} +{{- $subchart := "" }} +{{- $failOnNew := default true .failOnNew }} +{{- $chartName := default "" .chartName }} +{{- $passwordLength := default 10 .length }} +{{- $providedPasswordKey := include "common.utils.getKeyFromList" (dict "keys" .providedValues "context" $.context) }} +{{- $providedPasswordValue := include "common.utils.getValueFromKey" (dict "key" $providedPasswordKey "context" $.context) }} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data }} +{{- if $secretData }} + {{- if hasKey $secretData .key }} + {{- $password = index $secretData .key | quote }} + {{- else if $failOnNew }} + {{- printf "\nPASSWORDS ERROR: The secret \"%s\" does not contain the key \"%s\"\n" .secret .key | fail -}} + {{- end -}} +{{- else if $providedPasswordValue }} + {{- $password = $providedPasswordValue | toString | b64enc | quote }} +{{- else }} + + {{- if .context.Values.enabled }} + {{- $subchart = $chartName }} + {{- end -}} + + {{- $requiredPassword := dict "valueKey" $providedPasswordKey "secret" .secret "field" .key "subchart" $subchart "context" $.context -}} + {{- $requiredPasswordError := include "common.validations.values.single.empty" $requiredPassword -}} + {{- $passwordValidationErrors := list $requiredPasswordError -}} + {{- include "common.errors.upgrade.passwords.empty" (dict "validationErrors" $passwordValidationErrors "context" $.context) -}} + + {{- if .strong }} + {{- $subStr := list (lower (randAlpha 1)) (randNumeric 1) (upper (randAlpha 1)) | join "_" }} + {{- $password = randAscii $passwordLength }} + {{- $password = regexReplaceAllLiteral "\\W" $password "@" | substr 5 $passwordLength }} + {{- $password = printf "%s%s" $subStr $password | toString | shuffle | b64enc | quote }} + {{- else }} + {{- $password = randAlphaNum $passwordLength | b64enc | quote }} + {{- end }} +{{- end -}} +{{- printf "%s" $password -}} +{{- end -}} + +{{/* +Reuses the value from an existing secret, otherwise sets its value to a default value. + +Usage: +{{ include "common.secrets.lookup" (dict "secret" "secret-name" "key" "keyName" "defaultValue" .Values.myValue "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - key - String - Required - Name of the key in the secret. + - defaultValue - String - Required - The path to the validating value in the values.yaml, e.g: "mysql.password". Will pick first parameter with a defined value. + - context - Context - Required - Parent context. + +*/}} +{{- define "common.secrets.lookup" -}} +{{- $value := "" -}} +{{- $secretData := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret).data -}} +{{- if and $secretData (hasKey $secretData .key) -}} + {{- $value = index $secretData .key -}} +{{- else if .defaultValue -}} + {{- $value = .defaultValue | toString | b64enc -}} +{{- end -}} +{{- if $value -}} +{{- printf "%s" $value -}} +{{- end -}} +{{- end -}} + +{{/* +Returns whether a previous generated secret already exists + +Usage: +{{ include "common.secrets.exists" (dict "secret" "secret-name" "context" $) }} + +Params: + - secret - String - Required - Name of the 'Secret' resource where the password is stored. + - context - Context - Required - Parent context. +*/}} +{{- define "common.secrets.exists" -}} +{{- $secret := (lookup "v1" "Secret" (include "common.names.namespace" .context) .secret) }} +{{- if $secret }} + {{- true -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_storage.tpl b/command-service/helm-charts/flink/charts/common/templates/_storage.tpl new file mode 100644 index 00000000..16405a0f --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_storage.tpl @@ -0,0 +1,28 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Return the proper Storage Class +{{ include "common.storage.class" ( dict "persistence" .Values.path.to.the.persistence "global" $) }} +*/}} +{{- define "common.storage.class" -}} + +{{- $storageClass := .persistence.storageClass -}} +{{- if .global -}} + {{- if .global.storageClass -}} + {{- $storageClass = .global.storageClass -}} + {{- end -}} +{{- end -}} + +{{- if $storageClass -}} + {{- if (eq "-" $storageClass) -}} + {{- printf "storageClassName: \"\"" -}} + {{- else }} + {{- printf "storageClassName: %s" $storageClass -}} + {{- end -}} +{{- end -}} + +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_tplvalues.tpl b/command-service/helm-charts/flink/charts/common/templates/_tplvalues.tpl new file mode 100644 index 00000000..a8ed7637 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_tplvalues.tpl @@ -0,0 +1,38 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Renders a value that contains template perhaps with scope if the scope is present. +Usage: +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ ) }} +{{ include "common.tplvalues.render" ( dict "value" .Values.path.to.the.Value "context" $ "scope" $app ) }} +*/}} +{{- define "common.tplvalues.render" -}} +{{- $value := typeIs "string" .value | ternary .value (.value | toYaml) }} +{{- if contains "{{" (toJson .value) }} + {{- if .scope }} + {{- tpl (cat "{{- with $.RelativeScope -}}" $value "{{- end }}") (merge (dict "RelativeScope" .scope) .context) }} + {{- else }} + {{- tpl $value .context }} + {{- end }} +{{- else }} + {{- $value }} +{{- end }} +{{- end -}} + +{{/* +Merge a list of values that contains template after rendering them. +Merge precedence is consistent with http://masterminds.github.io/sprig/dicts.html#merge-mustmerge +Usage: +{{ include "common.tplvalues.merge" ( dict "values" (list .Values.path.to.the.Value1 .Values.path.to.the.Value2) "context" $ ) }} +*/}} +{{- define "common.tplvalues.merge" -}} +{{- $dst := dict -}} +{{- range .values -}} +{{- $dst = include "common.tplvalues.render" (dict "value" . "context" $.context "scope" $.scope) | fromYaml | merge $dst -}} +{{- end -}} +{{ $dst | toYaml }} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_utils.tpl b/command-service/helm-charts/flink/charts/common/templates/_utils.tpl new file mode 100644 index 00000000..bfbddf05 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_utils.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Print instructions to get a secret value. +Usage: +{{ include "common.utils.secret.getvalue" (dict "secret" "secret-name" "field" "secret-value-field" "context" $) }} +*/}} +{{- define "common.utils.secret.getvalue" -}} +{{- $varname := include "common.utils.fieldToEnvVar" . -}} +export {{ $varname }}=$(kubectl get secret --namespace {{ include "common.names.namespace" .context | quote }} {{ .secret }} -o jsonpath="{.data.{{ .field }}}" | base64 -d) +{{- end -}} + +{{/* +Build env var name given a field +Usage: +{{ include "common.utils.fieldToEnvVar" dict "field" "my-password" }} +*/}} +{{- define "common.utils.fieldToEnvVar" -}} + {{- $fieldNameSplit := splitList "-" .field -}} + {{- $upperCaseFieldNameSplit := list -}} + + {{- range $fieldNameSplit -}} + {{- $upperCaseFieldNameSplit = append $upperCaseFieldNameSplit ( upper . ) -}} + {{- end -}} + + {{ join "_" $upperCaseFieldNameSplit }} +{{- end -}} + +{{/* +Gets a value from .Values given +Usage: +{{ include "common.utils.getValueFromKey" (dict "key" "path.to.key" "context" $) }} +*/}} +{{- define "common.utils.getValueFromKey" -}} +{{- $splitKey := splitList "." .key -}} +{{- $value := "" -}} +{{- $latestObj := $.context.Values -}} +{{- range $splitKey -}} + {{- if not $latestObj -}} + {{- printf "please review the entire path of '%s' exists in values" $.key | fail -}} + {{- end -}} + {{- $value = ( index $latestObj . ) -}} + {{- $latestObj = $value -}} +{{- end -}} +{{- printf "%v" (default "" $value) -}} +{{- end -}} + +{{/* +Returns first .Values key with a defined value or first of the list if all non-defined +Usage: +{{ include "common.utils.getKeyFromList" (dict "keys" (list "path.to.key1" "path.to.key2") "context" $) }} +*/}} +{{- define "common.utils.getKeyFromList" -}} +{{- $key := first .keys -}} +{{- $reverseKeys := reverse .keys }} +{{- range $reverseKeys }} + {{- $value := include "common.utils.getValueFromKey" (dict "key" . "context" $.context ) }} + {{- if $value -}} + {{- $key = . }} + {{- end -}} +{{- end -}} +{{- printf "%s" $key -}} +{{- end -}} + +{{/* +Checksum a template at "path" containing a *single* resource (ConfigMap,Secret) for use in pod annotations, excluding the metadata (see #18376). +Usage: +{{ include "common.utils.checksumTemplate" (dict "path" "/configmap.yaml" "context" $) }} +*/}} +{{- define "common.utils.checksumTemplate" -}} +{{- $obj := include (print .context.Template.BasePath .path) .context | fromYaml -}} +{{ omit $obj "apiVersion" "kind" "metadata" | toYaml | sha256sum }} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_variables.tpl b/command-service/helm-charts/flink/charts/common/templates/_variables.tpl new file mode 100644 index 00000000..e5572991 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_variables.tpl @@ -0,0 +1,41 @@ +{{/* Define findkey function */}} +{{- define "common.variables.findkey" -}} +{{- $value := .value -}} +{{- $keys := .keys -}} +{{- $found := true -}} + +{{- range $key := $keys -}} + {{- if kindIs "map" $value -}} + {{- if hasKey $value $key -}} + {{- $value = index $value $key -}} + {{- else -}} + {{- $found = false -}} + {{- break -}} + {{- end -}} + {{- else -}} + {{- $found = false -}} + {{- break -}} + {{- end -}} +{{- end -}} + +{{- if $found -}} + {{- $value -}} +{{- else -}} + {{- "" -}} +{{- end -}} +{{- end -}} + +{{/* Define getmethekey function with precedence and input as a dot-delimited string using findkey */}} +{{- define "common.variables.variableGlobal" -}} +{{- $root := first . -}} +{{- $keyString := index (rest .) 0 -}} +{{- $keys := splitList "." $keyString -}} + +{{- $value := include "common.variables.findkey" (dict "value" $root.Values "keys" $keys) -}} + +{{- if (not $value) -}} + {{- $value = include "common.variables.findkey" (dict "value" $root.Values.global "keys" $keys) -}} +{{- end -}} + +{{- $value -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/_warnings.tpl b/command-service/helm-charts/flink/charts/common/templates/_warnings.tpl new file mode 100644 index 00000000..66dffc1f --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/_warnings.tpl @@ -0,0 +1,19 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Warning about using rolling tag. +Usage: +{{ include "common.warnings.rollingTag" .Values.path.to.the.imageRoot }} +*/}} +{{- define "common.warnings.rollingTag" -}} + +{{- if and (contains "bitnami/" .repository) (not (.tag | toString | regexFind "-r\\d+$|sha256:")) }} +WARNING: Rolling tag detected ({{ .repository }}:{{ .tag }}), please note that it is strongly recommended to avoid using rolling tags in a production environment. ++info https://docs.bitnami.com/containers/how-to/understand-rolling-tags-containers/ +{{- end }} + +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_cassandra.tpl b/command-service/helm-charts/flink/charts/common/templates/validations/_cassandra.tpl new file mode 100644 index 00000000..eda9aada --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/validations/_cassandra.tpl @@ -0,0 +1,77 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Cassandra required passwords are not empty. + +Usage: +{{ include "common.validations.values.cassandra.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where Cassandra values are stored, e.g: "cassandra-passwords-secret" + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.cassandra.passwords" -}} + {{- $existingSecret := include "common.cassandra.values.existingSecret" . -}} + {{- $enabled := include "common.cassandra.values.enabled" . -}} + {{- $dbUserPrefix := include "common.cassandra.values.key.dbUser" . -}} + {{- $valueKeyPassword := printf "%s.password" $dbUserPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "cassandra-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.cassandra.values.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.cassandra.dbUser.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.dbUser.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled cassandra. + +Usage: +{{ include "common.cassandra.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.cassandra.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.cassandra.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key dbUser + +Usage: +{{ include "common.cassandra.values.key.dbUser" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether Cassandra is used as subchart or not. Default: false +*/}} +{{- define "common.cassandra.values.key.dbUser" -}} + {{- if .subchart -}} + cassandra.dbUser + {{- else -}} + dbUser + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_mariadb.tpl b/command-service/helm-charts/flink/charts/common/templates/validations/_mariadb.tpl new file mode 100644 index 00000000..17d83a2f --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/validations/_mariadb.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MariaDB required passwords are not empty. + +Usage: +{{ include "common.validations.values.mariadb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MariaDB values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mariadb.passwords" -}} + {{- $existingSecret := include "common.mariadb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mariadb.values.enabled" . -}} + {{- $architecture := include "common.mariadb.values.architecture" . -}} + {{- $authPrefix := include "common.mariadb.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mariadb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mariadb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mariadb-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mariadb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mariadb. + +Usage: +{{ include "common.mariadb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mariadb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mariadb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mariadb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mariadb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mariadb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MariaDB is used as subchart or not. Default: false +*/}} +{{- define "common.mariadb.values.key.auth" -}} + {{- if .subchart -}} + mariadb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_mongodb.tpl b/command-service/helm-charts/flink/charts/common/templates/validations/_mongodb.tpl new file mode 100644 index 00000000..bbb445b8 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/validations/_mongodb.tpl @@ -0,0 +1,113 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MongoDB® required passwords are not empty. + +Usage: +{{ include "common.validations.values.mongodb.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MongoDB® values are stored, e.g: "mongodb-passwords-secret" + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mongodb.passwords" -}} + {{- $existingSecret := include "common.mongodb.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mongodb.values.enabled" . -}} + {{- $authPrefix := include "common.mongodb.values.key.auth" . -}} + {{- $architecture := include "common.mongodb.values.architecture" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyDatabase := printf "%s.database" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicaSetKey := printf "%s.replicaSetKey" $authPrefix -}} + {{- $valueKeyAuthEnabled := printf "%s.enabled" $authPrefix -}} + + {{- $authEnabled := include "common.utils.getValueFromKey" (dict "key" $valueKeyAuthEnabled "context" .context) -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") (eq $authEnabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mongodb-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- $valueDatabase := include "common.utils.getValueFromKey" (dict "key" $valueKeyDatabase "context" .context) }} + {{- if and $valueUsername $valueDatabase -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mongodb-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replicaset") -}} + {{- $requiredReplicaSetKey := dict "valueKey" $valueKeyReplicaSetKey "secret" .secret "field" "mongodb-replica-set-key" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicaSetKey -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mongodb.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDb is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mongodb. + +Usage: +{{ include "common.mongodb.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mongodb.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mongodb.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mongodb.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.key.auth" -}} + {{- if .subchart -}} + mongodb.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mongodb.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MongoDB® is used as subchart or not. Default: false +*/}} +{{- define "common.mongodb.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mongodb.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_mysql.tpl b/command-service/helm-charts/flink/charts/common/templates/validations/_mysql.tpl new file mode 100644 index 00000000..ca3953f8 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/validations/_mysql.tpl @@ -0,0 +1,108 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate MySQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.mysql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where MySQL values are stored, e.g: "mysql-passwords-secret" + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.mysql.passwords" -}} + {{- $existingSecret := include "common.mysql.values.auth.existingSecret" . -}} + {{- $enabled := include "common.mysql.values.enabled" . -}} + {{- $architecture := include "common.mysql.values.architecture" . -}} + {{- $authPrefix := include "common.mysql.values.key.auth" . -}} + {{- $valueKeyRootPassword := printf "%s.rootPassword" $authPrefix -}} + {{- $valueKeyUsername := printf "%s.username" $authPrefix -}} + {{- $valueKeyPassword := printf "%s.password" $authPrefix -}} + {{- $valueKeyReplicationPassword := printf "%s.replicationPassword" $authPrefix -}} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $requiredRootPassword := dict "valueKey" $valueKeyRootPassword "secret" .secret "field" "mysql-root-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRootPassword -}} + + {{- $valueUsername := include "common.utils.getValueFromKey" (dict "key" $valueKeyUsername "context" .context) }} + {{- if not (empty $valueUsername) -}} + {{- $requiredPassword := dict "valueKey" $valueKeyPassword "secret" .secret "field" "mysql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPassword -}} + {{- end -}} + + {{- if (eq $architecture "replication") -}} + {{- $requiredReplicationPassword := dict "valueKey" $valueKeyReplicationPassword "secret" .secret "field" "mysql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.mysql.values.auth.existingSecret" (dict "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.auth.existingSecret" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.auth.existingSecret | quote -}} + {{- else -}} + {{- .context.Values.auth.existingSecret | quote -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled mysql. + +Usage: +{{ include "common.mysql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.mysql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.mysql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for architecture + +Usage: +{{ include "common.mysql.values.architecture" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.architecture" -}} + {{- if .subchart -}} + {{- .context.Values.mysql.architecture -}} + {{- else -}} + {{- .context.Values.architecture -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key auth + +Usage: +{{ include "common.mysql.values.key.auth" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether MySQL is used as subchart or not. Default: false +*/}} +{{- define "common.mysql.values.key.auth" -}} + {{- if .subchart -}} + mysql.auth + {{- else -}} + auth + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_postgresql.tpl b/command-service/helm-charts/flink/charts/common/templates/validations/_postgresql.tpl new file mode 100644 index 00000000..8c9aa570 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/validations/_postgresql.tpl @@ -0,0 +1,134 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate PostgreSQL required passwords are not empty. + +Usage: +{{ include "common.validations.values.postgresql.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where postgresql values are stored, e.g: "postgresql-passwords-secret" + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.postgresql.passwords" -}} + {{- $existingSecret := include "common.postgresql.values.existingSecret" . -}} + {{- $enabled := include "common.postgresql.values.enabled" . -}} + {{- $valueKeyPostgresqlPassword := include "common.postgresql.values.key.postgressPassword" . -}} + {{- $valueKeyPostgresqlReplicationEnabled := include "common.postgresql.values.key.replicationPassword" . -}} + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + {{- $requiredPostgresqlPassword := dict "valueKey" $valueKeyPostgresqlPassword "secret" .secret "field" "postgresql-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlPassword -}} + + {{- $enabledReplication := include "common.postgresql.values.enabled.replication" . -}} + {{- if (eq $enabledReplication "true") -}} + {{- $requiredPostgresqlReplicationPassword := dict "valueKey" $valueKeyPostgresqlReplicationEnabled "secret" .secret "field" "postgresql-replication-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredPostgresqlReplicationPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to decide whether evaluate global values. + +Usage: +{{ include "common.postgresql.values.use.global" (dict "key" "key-of-global" "context" $) }} +Params: + - key - String - Required. Field to be evaluated within global, e.g: "existingSecret" +*/}} +{{- define "common.postgresql.values.use.global" -}} + {{- if .context.Values.global -}} + {{- if .context.Values.global.postgresql -}} + {{- index .context.Values.global.postgresql .key | quote -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for existingSecret. + +Usage: +{{ include "common.postgresql.values.existingSecret" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.existingSecret" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "existingSecret" "context" .context) -}} + + {{- if .subchart -}} + {{- default (.context.Values.postgresql.existingSecret | quote) $globalValue -}} + {{- else -}} + {{- default (.context.Values.existingSecret | quote) $globalValue -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled postgresql. + +Usage: +{{ include "common.postgresql.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.postgresql.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key postgressPassword. + +Usage: +{{ include "common.postgresql.values.key.postgressPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.postgressPassword" -}} + {{- $globalValue := include "common.postgresql.values.use.global" (dict "key" "postgresqlUsername" "context" .context) -}} + + {{- if not $globalValue -}} + {{- if .subchart -}} + postgresql.postgresqlPassword + {{- else -}} + postgresqlPassword + {{- end -}} + {{- else -}} + global.postgresql.postgresqlPassword + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled.replication. + +Usage: +{{ include "common.postgresql.values.enabled.replication" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.enabled.replication" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.postgresql.replication.enabled -}} + {{- else -}} + {{- printf "%v" .context.Values.replication.enabled -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for the key replication.password. + +Usage: +{{ include "common.postgresql.values.key.replicationPassword" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether postgresql is used as subchart or not. Default: false +*/}} +{{- define "common.postgresql.values.key.replicationPassword" -}} + {{- if .subchart -}} + postgresql.replication.password + {{- else -}} + replication.password + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_redis.tpl b/command-service/helm-charts/flink/charts/common/templates/validations/_redis.tpl new file mode 100644 index 00000000..fc0d208d --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/validations/_redis.tpl @@ -0,0 +1,81 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate Redis® required passwords are not empty. + +Usage: +{{ include "common.validations.values.redis.passwords" (dict "secret" "secretName" "subchart" false "context" $) }} +Params: + - secret - String - Required. Name of the secret where redis values are stored, e.g: "redis-passwords-secret" + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.validations.values.redis.passwords" -}} + {{- $enabled := include "common.redis.values.enabled" . -}} + {{- $valueKeyPrefix := include "common.redis.values.keys.prefix" . -}} + {{- $standarizedVersion := include "common.redis.values.standarized.version" . }} + + {{- $existingSecret := ternary (printf "%s%s" $valueKeyPrefix "auth.existingSecret") (printf "%s%s" $valueKeyPrefix "existingSecret") (eq $standarizedVersion "true") }} + {{- $existingSecretValue := include "common.utils.getValueFromKey" (dict "key" $existingSecret "context" .context) }} + + {{- $valueKeyRedisPassword := ternary (printf "%s%s" $valueKeyPrefix "auth.password") (printf "%s%s" $valueKeyPrefix "password") (eq $standarizedVersion "true") }} + {{- $valueKeyRedisUseAuth := ternary (printf "%s%s" $valueKeyPrefix "auth.enabled") (printf "%s%s" $valueKeyPrefix "usePassword") (eq $standarizedVersion "true") }} + + {{- if and (or (not $existingSecret) (eq $existingSecret "\"\"")) (eq $enabled "true") -}} + {{- $requiredPasswords := list -}} + + {{- $useAuth := include "common.utils.getValueFromKey" (dict "key" $valueKeyRedisUseAuth "context" .context) -}} + {{- if eq $useAuth "true" -}} + {{- $requiredRedisPassword := dict "valueKey" $valueKeyRedisPassword "secret" .secret "field" "redis-password" -}} + {{- $requiredPasswords = append $requiredPasswords $requiredRedisPassword -}} + {{- end -}} + + {{- include "common.validations.values.multiple.empty" (dict "required" $requiredPasswords "context" .context) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right value for enabled redis. + +Usage: +{{ include "common.redis.values.enabled" (dict "context" $) }} +*/}} +{{- define "common.redis.values.enabled" -}} + {{- if .subchart -}} + {{- printf "%v" .context.Values.redis.enabled -}} + {{- else -}} + {{- printf "%v" (not .context.Values.enabled) -}} + {{- end -}} +{{- end -}} + +{{/* +Auxiliary function to get the right prefix path for the values + +Usage: +{{ include "common.redis.values.key.prefix" (dict "subchart" "true" "context" $) }} +Params: + - subchart - Boolean - Optional. Whether redis is used as subchart or not. Default: false +*/}} +{{- define "common.redis.values.keys.prefix" -}} + {{- if .subchart -}}redis.{{- else -}}{{- end -}} +{{- end -}} + +{{/* +Checks whether the redis chart's includes the standarizations (version >= 14) + +Usage: +{{ include "common.redis.values.standarized.version" (dict "context" $) }} +*/}} +{{- define "common.redis.values.standarized.version" -}} + + {{- $standarizedAuth := printf "%s%s" (include "common.redis.values.keys.prefix" .) "auth" -}} + {{- $standarizedAuthValues := include "common.utils.getValueFromKey" (dict "key" $standarizedAuth "context" .context) }} + + {{- if $standarizedAuthValues -}} + {{- true -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_validations.tpl b/command-service/helm-charts/flink/charts/common/templates/validations/_validations.tpl new file mode 100644 index 00000000..31ceda87 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/templates/validations/_validations.tpl @@ -0,0 +1,51 @@ +{{/* +Copyright VMware, Inc. +SPDX-License-Identifier: APACHE-2.0 +*/}} + +{{/* vim: set filetype=mustache: */}} +{{/* +Validate values must not be empty. + +Usage: +{{- $validateValueConf00 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-00") -}} +{{- $validateValueConf01 := (dict "valueKey" "path.to.value" "secret" "secretName" "field" "password-01") -}} +{{ include "common.validations.values.empty" (dict "required" (list $validateValueConf00 $validateValueConf01) "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" +*/}} +{{- define "common.validations.values.multiple.empty" -}} + {{- range .required -}} + {{- include "common.validations.values.single.empty" (dict "valueKey" .valueKey "secret" .secret "field" .field "context" $.context) -}} + {{- end -}} +{{- end -}} + +{{/* +Validate a value must not be empty. + +Usage: +{{ include "common.validations.value.empty" (dict "valueKey" "mariadb.password" "secret" "secretName" "field" "my-password" "subchart" "subchart" "context" $) }} + +Validate value params: + - valueKey - String - Required. The path to the validating value in the values.yaml, e.g: "mysql.password" + - secret - String - Optional. Name of the secret where the validating value is generated/stored, e.g: "mysql-passwords-secret" + - field - String - Optional. Name of the field in the secret data, e.g: "mysql-password" + - subchart - String - Optional - Name of the subchart that the validated password is part of. +*/}} +{{- define "common.validations.values.single.empty" -}} + {{- $value := include "common.utils.getValueFromKey" (dict "key" .valueKey "context" .context) }} + {{- $subchart := ternary "" (printf "%s." .subchart) (empty .subchart) }} + + {{- if not $value -}} + {{- $varname := "my-value" -}} + {{- $getCurrentValue := "" -}} + {{- if and .secret .field -}} + {{- $varname = include "common.utils.fieldToEnvVar" . -}} + {{- $getCurrentValue = printf " To get the current value:\n\n %s\n" (include "common.utils.secret.getvalue" .) -}} + {{- end -}} + {{- printf "\n '%s' must not be empty, please add '--set %s%s=$%s' to the command.%s" .valueKey $subchart .valueKey $varname $getCurrentValue -}} + {{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/charts/common/values.yaml b/command-service/helm-charts/flink/charts/common/values.yaml new file mode 100644 index 00000000..c83b33f2 --- /dev/null +++ b/command-service/helm-charts/flink/charts/common/values.yaml @@ -0,0 +1,82 @@ +# Default values for common. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: nginx + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/command-service/helm-charts/flink/templates/NOTES.txt b/command-service/helm-charts/flink/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/flink/templates/_base_serviceAccount.tpl b/command-service/helm-charts/flink/templates/_base_serviceAccount.tpl new file mode 100644 index 00000000..dadfc22d --- /dev/null +++ b/command-service/helm-charts/flink/templates/_base_serviceAccount.tpl @@ -0,0 +1,4 @@ +{{- define "base.serviceaccountname" -}} + {{- $name := include "common.names.fullname" . }} + {{- default $name .Values.serviceAccount.name }} +{{- end }} diff --git a/command-service/helm-charts/flink/templates/_helpers.tpl b/command-service/helm-charts/flink/templates/_helpers.tpl new file mode 100644 index 00000000..e69de29b diff --git a/command-service/helm-charts/flink/templates/_image_flink.tpl b/command-service/helm-charts/flink/templates/_image_flink.tpl new file mode 100644 index 00000000..d0d7c2da --- /dev/null +++ b/command-service/helm-charts/flink/templates/_image_flink.tpl @@ -0,0 +1,15 @@ +{{/* {{- include "base.image.flink" dict ("context" $ "scope" $jobData) }} */}} +{{- define "base.image.flink" }} +{{- $context := .context }} +{{- $scope := .scope }} +{{- with $scope }} +{{- $registry := default $context.Values.global.image.registry .registry }} +{{- $image := printf "%s/%s" $registry .repository}} +{{- if .digest }} +{{- printf "%s@%s" $image .digest }} +{{- else }} +{{- $tag := default "latest" .tag }} +{{- printf "%s:%s" $image $tag }} +{{- end }} +{{- end }} +{{- end }} diff --git a/command-service/helm-charts/flink/templates/_namespace.tpl b/command-service/helm-charts/flink/templates/_namespace.tpl new file mode 100644 index 00000000..7af0b70b --- /dev/null +++ b/command-service/helm-charts/flink/templates/_namespace.tpl @@ -0,0 +1,13 @@ +# .Values.namespace will get overridden by .Values.global.namespace.chart-name +{{- define "base.namespace" -}} + {{- $chartName := .Chart.Name }} + {{- $namespace := default .Release.Namespace .Values.namespace }} + {{- if .Values.global }} + {{- with .Values.global.namespace }} + {{- if hasKey . $chartName }} + {{- $namespace = index . $chartName }} + {{- end }} + {{- end }} + {{- end }} + {{- $namespace | trunc 63 | trimSuffix "-" }} +{{- end }} diff --git a/command-service/helm-charts/flink/templates/configmap.yaml b/command-service/helm-charts/flink/templates/configmap.yaml new file mode 100644 index 00000000..b486db7f --- /dev/null +++ b/command-service/helm-charts/flink/templates/configmap.yaml @@ -0,0 +1,30 @@ +{{- $currentScope := .}} +{{- range $jobName, $jobData := .Values.flink_jobs }} +{{- if $jobData.enabled }} +{{- with $currentScope }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-config" $jobName }} + namespace: {{ include "base.namespace" . }} + labels: {{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: +{{ include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +data: + {{- range $configKey, $configValue := $jobData -}} + {{/* We don't have to create key value pair configmaps as files */}} + {{- if contains "\n" (toString $configValue) -}} + {{- $configKey | nindent 2 -}}: |- + {{- include "common.tplvalues.render" (dict "value" $configValue "context" $) | nindent 4 }} + {{- end -}} + {{- end -}} + {{- "baseconfig" | nindent 2 -}}: |- + {{- include "common.tplvalues.render" (dict "value" .Values.baseconfig "context" $) | nindent 4 }} + {{- "log4j_console_properties" | nindent 2 -}}: |- + {{- include "common.tplvalues.render" (dict "value" .Values.log4j_console_properties "context" $) | nindent 4 }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/command-service/helm-charts/flink/templates/deployment.yaml b/command-service/helm-charts/flink/templates/deployment.yaml new file mode 100644 index 00000000..cd7f5619 --- /dev/null +++ b/command-service/helm-charts/flink/templates/deployment.yaml @@ -0,0 +1,220 @@ +{{- $currentScope := .}} +{{- range $jobName, $jobData := .Values.flink_jobs }} +{{- if $jobData.enabled }} +{{- with $currentScope }} +{{ $component := "taskmanager" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-%s" $jobName $component }} + namespace: {{ include "base.namespace" $ }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + annotations: + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} + checksum/job-config: {{ $jobData | toYaml | sha256sum }} + {{- if .Values.commonAnnotations }} + {{ include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + component: {{ printf "%s-%s" $jobName $component }} + annotations: + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} + checksum/job-config: {{ $jobData | toYaml | sha256sum }} + spec: + {{- if .Values.serviceAccount.create}} + serviceAccountName: {{ include "base.serviceaccountname" . }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- $imagePullSecrets := default .Values.imagePullSecrets $jobData.imagePullSecrets }} + {{- with $imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ $jobName }}-taskmanager + image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} + imagePullPolicy: {{ default .Values.imagePullPolicy "IfNotPresent" }} + workingDir: {{ .Values.taskmanager.flink_work_dir }} + command: ["/opt/flink/bin/taskmanager.sh"] + args: ["start-foreground", + {{- if eq .Values.checkpoint_store_type "azure" }} + "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}", + {{- end }} + {{- if and (eq .Values.checkpoint_store_type "s3") (ne .Values.s3_auth_type "serviceAccount") }} + "-Ds3.access-key={{ .Values.s3_access_key }}", + "-Ds3.secret-key={{ .Values.s3_secret_key }}", + "-Ds3.endpoint={{ .Values.s3_endpoint }}", + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcs" }} + "-Dgoogle.cloud.auth.service.account.enable=true", + {{- end }} + "-Dweb.submit.enable=false", + "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", + "-Dmetrics.reporter.prom.host={{ $jobName }}-taskmanager", + "-Dmetrics.reporter.prom.port=9251-9260", + "-Djobmanager.rpc.address={{ $jobName }}-jobmanager", + "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"] + ports: + - containerPort: {{ .Values.taskmanager.rpc_port }} + name: rpc + {{- toYaml (index $jobData "resources" $component )| nindent 10 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + volumeMounts: + - name: flink-config-volume + mountPath: /opt/flink/conf/flink-conf.yaml + subPath: flink-conf.yaml + - name: flink-config-volume + mountPath: /data/flink/conf/baseconfig.conf + subPath: baseconfig.conf + - name: flink-config-volume + mountPath: /data/flink/conf/{{ $jobName }}.conf + subPath: {{ $jobName }}.conf + - name: flink-config-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties + volumes: + - name: flink-config-volume + configMap: + name: {{ $jobName }}-config + items: + - key: flink-conf + path: flink-conf.yaml + - key: baseconfig + path: baseconfig.conf + - key: config + path: {{ $jobName }}.conf + - key: log4j_console_properties + path: log4j-console.properties +{{ $component := "jobmanager" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ printf "%s-%s" $jobName $component }} + namespace: {{ include "base.namespace" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + annotations: + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} + checksum/job-config: {{ $jobData | toYaml | sha256sum }} + {{- if .Values.commonAnnotations }} + {{ include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} + component: {{ printf "%s-%s" $jobName $component }} + annotations: + checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} + checksum/job-config: {{ $jobData | toYaml | sha256sum }} + spec: + serviceAccountName: {{ include "base.serviceaccountname" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- $imagePullSecrets := default .Values.imagePullSecrets $jobData.imagePullSecrets }} + {{- with $imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: {{ $jobName }}-jobmanager + image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} + imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + workingDir: /opt/flink + command: + - "/bin/sh" + - "-c" + - "/usr/bin/python3.11 -c 'print(5**3)'" + - "/bin/sh" + - "-c" + - "/opt/flink/bin/standalone-job.sh" + args: ["start-foreground", + "--job-classname={{ index $jobData "job_classname" }}", + {{- if eq .Values.checkpoint_store_type "azure" }} + "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}", + {{- end }} + {{- if and (eq .Values.checkpoint_store_type "s3") (ne .Values.s3_auth_type "serviceAccount") }} + "-Ds3.access-key={{ .Values.s3_access_key }}", + "-Ds3.secret-key={{ .Values.s3_secret_key }}", + "-Ds3.endpoint={{ .Values.s3_endpoint }}", + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcs" }} + "-Dgoogle.cloud.auth.service.account.enable=true", + {{- end }} + "-Dweb.submit.enable=false", + "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", + "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}", + "-Djobmanager.rpc.address={{ $jobName }}-jobmanager", + "-Djobmanager.rpc.port={{ .Values.jobmanager.rpc_port }}", + "-Dparallelism.default=1", + "-Dblob.server.port={{ .Values.jobmanager.blob_port }}", + "-Dqueryable-state.server.ports={{ .Values.jobmanager.query_port }}", + "--config.file.path", + "/data/flink/conf/{{ $jobName }}.conf"] + ports: + {{- range .Values.service.ports }} + - name: {{ .name }} + containerPort: {{ .targetPort }} + {{- end }} + {{- toYaml (index $jobData "resources" $component )| nindent 10 }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + volumeMounts: + - name: flink-config-volume + mountPath: /opt/flink/conf/flink-conf.yaml + subPath: flink-conf.yaml + - name: flink-config-volume + mountPath: /data/flink/conf/baseconfig.conf + subPath: baseconfig.conf + - name: flink-config-volume + mountPath: /data/flink/conf/{{ $jobName }}.conf + subPath: {{ $jobName }}.conf + - name: flink-config-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties + volumes: + - name: flink-config-volume + configMap: + name: {{ $jobName }}-config + items: + - key: flink-conf + path: flink-conf.yaml + - key: baseconfig + path: baseconfig.conf + - key: config + path: {{ $jobName }}.conf + - key: log4j_console_properties + path: log4j-console.properties +{{- end }} +{{- end}} +{{- end}} \ No newline at end of file diff --git a/command-service/helm-charts/flink/templates/hpa.yaml b/command-service/helm-charts/flink/templates/hpa.yaml new file mode 100644 index 00000000..a9600ffb --- /dev/null +++ b/command-service/helm-charts/flink/templates/hpa.yaml @@ -0,0 +1,20 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "common.names.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- toYaml .Values.autoscaling.metrics | nindent 4 }} +{{- end }} diff --git a/command-service/helm-charts/flink/templates/ingress.yaml b/command-service/helm-charts/flink/templates/ingress.yaml new file mode 100644 index 00000000..7a01585c --- /dev/null +++ b/command-service/helm-charts/flink/templates/ingress.yaml @@ -0,0 +1,24 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "base.namespace" . }} + annotations: + {{- toYaml .Values.ingress.annotations | nindent 4 }} +spec: + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host }} + http: + paths: + {{- range .paths }} + - path: {{ . }} + backend: + service: + name: {{ include "common.names.fullname" $ }} + port: + name: http + {{- end }} + {{- end }} +{{- end }} diff --git a/command-service/helm-charts/flink/templates/service.yaml b/command-service/helm-charts/flink/templates/service.yaml new file mode 100644 index 00000000..e1c3f711 --- /dev/null +++ b/command-service/helm-charts/flink/templates/service.yaml @@ -0,0 +1,42 @@ +{{- range $jobName, $jobData := .Values.flink_jobs }} +{{- if $jobData.enabled }} +{{- $components := list "jobmanager" "taskmanager" }} +{{- range $component := $components }} +--- +apiVersion: v1 +kind: Service +metadata: + namespace: {{ include "base.namespace" $}} + name: {{ $jobName }}-{{ $component }} + labels: + {{- include "common.labels.standard" (dict "customLabels" $.Values.commonLabels "context" $) | nindent 4 }} + component: {{ printf "%s-%s" $jobName $component }} + + {{- if $.Values.commonAnnotations }} + annotations: + {{- include "common.tplvalues.render" (dict "value" $.Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + {{- if eq $component "jobmanager" }} + - name: rpc + port: {{ $.Values.jobmanager.rpc_port }} + - name: blob + port: {{ $.Values.jobmanager.blob_port }} + - name: query + port: {{ $.Values.jobmanager.query_port }} + - name: ui + port: {{ $.Values.jobmanager.ui_port }} + - name: prom + port: {{ $.Values.jobmanager.prom_port }} + {{- else if eq $component "taskmanager" }} + - name: prom + port: {{ $.Values.taskmanager.prom_port }} + {{- end }} + selector: + app.kubernetes.io/component: {{ $jobName }}-{{ $component }} +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/command-service/helm-charts/flink/templates/serviceaccount.yaml b/command-service/helm-charts/flink/templates/serviceaccount.yaml new file mode 100644 index 00000000..d26e8536 --- /dev/null +++ b/command-service/helm-charts/flink/templates/serviceaccount.yaml @@ -0,0 +1,11 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "base.serviceaccountname" . }} + namespace: {{ include "base.namespace" . }} + {{- if or .Values.serviceAccount.annotations .Values.commonAnnotations }} + {{- $annotations := include "common.tplvalues.merge" ( dict "values" ( list .Values.serviceAccount.annotations .Values.commonAnnotations ) "context" . ) }} + annotations: {{- include "common.tplvalues.render" ( dict "value" $annotations "context" $) | nindent 4 }} + {{- end }} +{{- end }} diff --git a/command-service/helm-charts/flink/templates/servicemonitor.yaml b/command-service/helm-charts/flink/templates/servicemonitor.yaml new file mode 100644 index 00000000..8f7160c9 --- /dev/null +++ b/command-service/helm-charts/flink/templates/servicemonitor.yaml @@ -0,0 +1,31 @@ +{{- $currentScope := .}} +{{- range $jobName, $jobData := .Values.flink_jobs }} +{{- range $key, $smData := $jobData.serviceMonitor }} +{{- if $jobData.enabled}} +{{- if $smData.enabled}} +{{- with $currentScope }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ printf "%s-%s" $jobName $key }} + namespace: {{ include "base.namespace" . }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ $smData.jobLabel }} + selector: + matchLabels: + {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 6 }} + endpoints: + - port: {{ $smData.port }} # the name of the port in your service, assuming the primary service port is named 'http' in this example. + interval: {{ $smData.interval }} + scrapeTimeout: {{ $smData.scrapeTimeout }} + honorLabels: {{ $smData.honorLabels }} +{{- end}} +{{- end}} +{{- end}} +{{- end}} +{{- end}} \ No newline at end of file diff --git a/command-service/helm-charts/flink/values.yaml b/command-service/helm-charts/flink/values.yaml new file mode 100644 index 00000000..a7bd26e3 --- /dev/null +++ b/command-service/helm-charts/flink/values.yaml @@ -0,0 +1,611 @@ +# vim: set fdm=indent: +nameOverride: "" +fullnameOverride: "" + +replicaCount: 1 + +namespace: "flink" +commonLabels: + system.processing: "true" + release: monitoring + app: flink + + +# repository: sunbirded.azurecr.io/data-pipeline +# tag: "release-5.2.0_RC1_2c615f8_12" +# docker pull sunbirded.azurecr.io/sunbird-datapipeline:release-4.9.0_RC4_1 +registry: surabhi1510 +repository: flink-python-3.11 +tag: 1.0.0 +imagePullSecrets: [] + +## Databases +global: + redis: + host: redis-denorm.redis.svc.cluster.local + port: 6379 + cassandra: + host: localhost + port: 9042 + kafka: + host: "kafka-headless.kafka.svc.cluster.local" + port: 9092 + zookeeper: + host: "zookeeper-headless.kafka.svc.cluster.local" + port: 2181 + image: + registry: "" + +podAnnotations: {} + +podSecurityContext: + runAsNonRoot: true + runAsUser: 9999 + fsGroup: 0 + +securityContext: + {} + # readOnlyRootFilesystem: false + # capabilities: + # drop: + # - ALL + +service: + type: ClusterIP + ports: + - name: http + port: 8081 + targetPort: 8081 + +ingress: + enabled: false + annotations: {} + hosts: + - host: chart-example.local + paths: + - / + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + - type: Resource + resource: + name: memory + target: + type: Utilization + averageUtilization: 80 + +nodeSelector: {} +tolerations: [] +affinity: {} + +configmap: + enabled: false + mountPath: /config + +serviceAccount: + create: true + name: "" + +# Example values.yaml structure +initContainers: + {} + # - name: init-myservice + # image: busybox:1.28 + # command: ['sh', '-c', "until nslookup kubernetes.$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace).svc.cluster.local; do echo waiting for myservice; sleep 2; done"] + +sidecars: + {} + # - name: log-reader # Sidecar container + # image: busybox # Use another busybox image + # command: ["/bin/sh"] # Override the default command + # args: ["-c", "tail -f /var/log/app.log"] # Run a shell script that tails the log file + +opa: + enabled: false + +jobmanager: + rpc_port: 6123 + blob_port: 6124 + query_port: 6125 + ui_port: 8081 + prom_port: 9250 + heap_memory: 1024 + +rest_port: 80 +resttcp_port: 8081 + +taskmanager: + prom_port: 9251 + rpc_port: 6122 + heap_memory: 1024 + replicas: 1 + cpu_requests: 0.3 + +checkpoint_store_type: null + +# AWS S3 Details +s3_auth_type: "serviceAccount" +s3_access_key: "" +s3_secret_key: "" +s3_endpoint: "" +s3_path_style_access: "" + +# Azure Container Details +azure_account: "azure-test" +azure_secret: "azure-secret" + +# Azure Container Details +cloud_storage_flink_bucketname: flink-state-backend +cloud_storage_content_bucketname: sunbird-content-dev +cert_container_name: dev-e-credentials +# cloud_storage_endpoint: https://{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net + +flink_dp_storage_container: "" +checkpoint_interval: 60000 +checkpoint_pause_between_seconds: 5000 +checkpoint_compression_enabled: true +restart_attempts: 3 +restart_delay: 30000 # in milli-seconds +producer_max_request_size: 1572864 +producer_batch_size: 98304 +producer_linger_ms: 10 +producer_compression: snappy +extractor_event_max_size: 1048576 # Max is only 1MB +extractor_max_request_size: 5242880 +extractor_consumer_parallelism: 1 +extractor_operators_parallelism: 1 +telemetry_extractor_key_expiry_seconds: 3600 +ingest_router_consumer_parallelism: 1 +ingest_router_operators_parallelism: 1 +raw_router_consumer_parallelism: 1 +raw_router_downstream_parallelism: 1 + +content_port: 6379 +device_port: 6380 +user_port: 6381 +dialcode_port: 6382 + +pipeline_preprocessor_consumer_parallelism: 1 +pipeline_preprocessor_operators_parallelism: 1 +portal_id: ".sunbird.portal" +desktop_id: ".sunbird.desktop" +pipeline_preprocessor_key_expiry_seconds: 3600 + +denorm_secondary_window_count: 30 +denorm_secondary_window_shards: 1400 +denorm_primary_window_count: 30 +denorm_primary_window_shards: 1400 + +denorm_summary_window_count: 5 +denorm_summary_window_shards: 1400 + +denorm_secondary_consumer_parallelism: 1 +telemetry_denorm_secondary_operators_parallelism: 1 +denorm_primary_consumer_parallelism: 1 +telemetry_denorm_primary_operators_parallelism: 1 + +### summary-denormalization related vars +summary_denorm_consumer_parallelism: 1 +summary_denorm_operators_parallelism: 1 +summary_denorm_duplication_key_expiry_seconds: 3600 +summary_denorm_key_expiry_seconds: 3600 + +### De-normalization related vars +denorm_consumer_parallelism: 1 +telemetry_denorm_operators_parallelism: 1 +de_normalization_duplicationstore_key_expiry_seconds: 3600 +de_normalization_key_expiry_seconds: 3600 +denorm_window_count: 30 +denorm_window_shards: 1400 + +### Druid-validator related vars +druid_validator_consumer_parallelism: 1 +druid_validator_operators_parallelism: 1 +druid_validator_key_expiry_seconds: 3600 +druid_validation_enabled: true +druid_deduplication_enabled: true + +### error-denormalization related vars +error_denorm_consumer_parallelism: 1 +error_denorm_operators_parallelism: 1 + +### Device-profile-updater related vars +deviceprofile_parallelism: 1 +device_profile_updater_key_expiry_seconds: 3600 +device_profile_table: "_device_profile" + +### content-cache-updater +# dialcode_api_url: {{ .Values.global.proto }}://{{ .Values.global.domain_name }}/{{ .Values.dialcode_endpoint }} +dialcode_api_auth_key: "" + +#(user read api details) +user_read_api_endpoint: "/private/user/v1/read/" +user_read_api_url: "learner-service:9000" + +### User-cache-updater related vars +usercache_updater_parallelism: 1 +user_cache_updater_key_expiry_seconds: 3600 +middleware_cassandra_keyspace: sunbird +middleware_cassandra_user_table: user +middleware_cassandra_location_table: location + +postgres: + max_connections: 2 + sslmode: false + db_name: analytics + db_port: 5432 + +checkpointing: + enabled: false + statebackend: null + +log4j_console_properties: | + # This affects logging for both user code and Flink + rootLogger.level = INFO + rootLogger.appenderRef.console.ref = ConsoleAppender + rootLogger.appenderRef.rolling.ref = RollingFileAppender + + # Uncomment this if you want to _only_ change Flink's logging + logger.flink.name = org.apache.flink + logger.flink.level = INFO + + # The following lines keep the log level of common libraries/connectors on + # log level INFO. The root logger does not override this. You have to manually + # change the log levels here. + logger.akka.name = akka + logger.akka.level = ERROR + logger.kafka.name= org.apache.kafka + logger.kafka.level = ERROR + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = ERROR + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = ERROR + + # Log all infos to the console + appender.console.name = ConsoleAppender + appender.console.type = CONSOLE + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + + # Log all infos in the given rolling file + appender.rolling.name = RollingFileAppender + appender.rolling.type = RollingFile + appender.rolling.append = false + appender.rolling.fileName = ${sys:log.file} + appender.rolling.filePattern = ${sys:log.file}.%i + appender.rolling.layout.type = PatternLayout + appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + appender.rolling.policies.type = Policies + appender.rolling.policies.size.type = SizeBasedTriggeringPolicy + appender.rolling.policies.size.size=10MB + appender.rolling.strategy.type = DefaultRolloverStrategy + appender.rolling.strategy.max = 5 + + # Suppress the irrelevant (wrong) warnings from the Netty channel handler + logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline + logger.netty.level = OFF + +baseconfig: | + kafka { + broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" + producer.broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" + consumer.broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" + zookeeper = "{{ .Values.global.zookeeper.host }}:{{ .Values.global.zookeeper.port }}" + producer { + max-request-size = 1572864 + batch.size = 98304 + linger.ms = 10 + compression = "snappy" + } + output.system.event.topic = ${job.env}".system.events" + + output.failed.topic = ${job.env}".failed" + } + job { + env = "{{ .Values.global.env }}" + enable.distributed.checkpointing = {{ .Values.checkpointing.enabled }} + statebackend { + base.url = "{{ .Values.checkpointing.statebackend }}" + } + } + + task { + parallelism = 1 + consumer.parallelism = 1 + checkpointing.interval = 10000 + checkpointing.pause.between.seconds = 10000 + restart-strategy.attempts = 3 + restart-strategy.delay = 30000 # in milli-seconds + } + + redis.connection.timeout = 100 + redis { + host = "{{ .Values.global.redis_dedup.host }}" + port = {{ .Values.global.redis_dedup.port }} + } + + redis-meta { + host = "{{ .Values.global.redis_denorm.host }}" + port = {{ .Values.global.redis_denorm.port }} + } + + postgres { + host = "{{ .Values.global.postgresql.host }}" + port = "{{ .Values.global.postgresql.port }}" + maxConnections = "{{ .Values.postgres.max_connections }}" + sslMode = "{{ .Values.postgres.sslmode }}" + user = "{{ .Values.global.postgresql.obsrv.user }}" + password = "{{ .Values.global.postgresql.obsrv.password }}" + database = "{{ .Values.global.postgresql.obsrv.name }}" + } + + lms-cassandra { + host = "{{ .Values.global.cassandra.host }}" + port = "{{ .Values.global.cassandra.port }}" + } + +# !!! Don't override the resources here. It's just a template +# Proper way to override resouce is to +# flink_jobs: +# master-data-processor: +# resources: +# taskmanager: +# resources: +# requests: +# cpu: 100m +# memory: 100Mi +# limits: +# cpu: 1 +# memory: 1024Mi +# jobmanager: +# resources: +# requests: +# cpu: 100m +# memory: 100Mi +# limits: +# cpu: 1 +# memory: 1024Mi +dummy_flink_resource: &flink_resources + taskmanager: + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 1 + memory: 1024Mi + jobmanager: + resources: + requests: + cpu: 100m + memory: 100Mi + limits: + cpu: 1 + memory: 1024Mi + +serviceMonitor: &serviceMonitor + jobmanager: + enabled: true + interval: 30s + scrapeTimeout: 10s + labels: {} # additional labels e.g. release: prometheus + honorLabels: true + jobLabel: "app.kubernetes.io/name" + port: prom + taskmanager: + enabled: true + interval: 30s + scrapeTimeout: 10s + labels: {} # additional labels e.g. release: prometheus + honorLabels: true + jobLabel: "app.kubernetes.io/name" + port: prom + +flink_jobs: + kafka-connector: + enabled: true + + registry: surabhi1510 + repository: flink-python-3.11 + tag: 1.0.0 + imagePullSecrets: [] + + job_classname: in.sanketika.obsrv.pipeline.task.MasterDataProcessorStreamTask + resources: *flink_resources + serviceMonitor: *serviceMonitor + + config: |+ + include file("/data/flink/conf/baseconfig.conf") + + flink-conf: |+ + jobmanager.memory.flink.size: 1024m + taskmanager.memory.flink.size: 1024m + taskmanager.numberOfTaskSlots: 2 + jobmanager.numberOfTaskSlots: 1 + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + state.savepoints.dir: file:///tmp + + unified-pipeline: + enabled: false + + registry: sanketikahub + repository: unified-pipeline + tag: 1.0.0-GA + imagePullSecrets: [] + + job_classname: in.sanketika.obsrv.pipeline.task.UnifiedPipelineStreamTask + resources: *flink_resources + serviceMonitor: *serviceMonitor + + config: |+ + include file("/data/flink/conf/baseconfig.conf") + kafka { + input.topic = ${job.env}".ingest" + output.raw.topic = ${job.env}".raw" + output.extractor.duplicate.topic = ${job.env}".failed" + output.batch.failed.topic = ${job.env}".failed" + event.max.size = "1048576" # Max is only 1MB + output.invalid.topic = ${job.env}".failed" + output.unique.topic = ${job.env}".unique" + output.duplicate.topic = ${job.env}".failed" + output.denorm.topic = ${job.env}".denorm" + output.denorm.failed.topic = ${job.env}".failed" + output.transform.topic = ${job.env}".transform" + output.transform.failed.topic = ${job.env}".transform.failed" + stats.topic = ${job.env}".stats" + groupId = ${job.env}"-unified-pipeline-group" + producer { + max-request-size = 5242880 + } + } + + task { + window.time.in.seconds = 5 + window.count = 30 + window.shards = 1400 + consumer.parallelism = 2 + downstream.operators.parallelism = 2 + } + + redis { + database { + extractor.duplication.store.id = 1 + preprocessor.duplication.store.id = 2 + key.expiry.seconds = 3600 + } + } + flink-conf: |+ + jobmanager.memory.flink.size: 1024m + taskmanager.memory.flink.size: 1024m + taskmanager.numberOfTaskSlots: 2 + jobmanager.numberOfTaskSlots: 1 + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + state.savepoints.dir: file:///tmp + + transformer-ext: + enabled: false + + registry: sanketikahub + repository: transformer-ext + tag: 1.0.0-GA + imagePullSecrets: [] + + job_classname: in.sanketika.obsrv.transformer.task.TransformerStreamTask + resources: *flink_resources + serviceMonitor: *serviceMonitor + + config: |+ + include file("/data/flink/conf/baseconfig.conf") + kafka { + input.topic = ${job.env}".denorm" + output.transform.topic = ${job.env}".transform" + output.transform.failed.topic = ${job.env}".transform.failed" + groupId = ${job.env}"-transformer-group" + producer { + max-request-size = 5242880 + } + } + + task { + consumer.parallelism = 1 + downstream.operators.parallelism = 1 + } + flink-conf: |+ + jobmanager.memory.flink.size: 1024m + taskmanager.memory.flink.size: 1024m + taskmanager.numberOfTaskSlots: 2 + jobmanager.numberOfTaskSlots: 1 + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + state.savepoints.dir: file:///tmp + + master-data-processor-ext: + enabled: false + + registry: sanketikahub + repository: master-data-processor-ext + tag: 1.0.0-GA + imagePullSecrets: [] + + job_classname: in.sanketika.obsrv.pipeline.task.MasterDataProcessorStreamTask + resources: *flink_resources + serviceMonitor: *serviceMonitor + + config: |+ + include file("/data/flink/conf/baseconfig.conf") + kafka { + input.topic = ${job.env}".masterdata.ingest" + output.raw.topic = ${job.env}".masterdata.raw" + output.extractor.duplicate.topic = ${job.env}".masterdata.failed" + output.failed.topic = ${job.env}".masterdata.failed" + output.batch.failed.topic = ${job.env}".masterdata.failed" + event.max.size = "1048576" # Max is only 1MB + output.invalid.topic = ${job.env}".masterdata.failed" + output.unique.topic = ${job.env}".masterdata.unique" + output.duplicate.topic = ${job.env}".masterdata.failed" + output.denorm.topic = ${job.env}".masterdata.denorm" + output.transform.topic = ${job.env}".masterdata.transform" + output.transform.failed.topic = ${job.env}".masterdata.transform.failed" + stats.topic = ${job.env}".masterdata.stats" + groupId = ${job.env}"-masterdata-pipeline-group" + + producer { + max-request-size = 5242880 + } + } + + task { + window.time.in.seconds = 5 + window.count = 30 + window.shards = 1400 + consumer.parallelism = 1 + downstream.operators.parallelism = 1 + } + + redis { + database { + extractor.duplication.store.id = 1 + preprocessor.duplication.store.id = 2 + key.expiry.seconds = 3600 + } + } + + dataset.type = "master-dataset" + flink-conf: |+ + jobmanager.memory.flink.size: 1024m + taskmanager.memory.flink.size: 1024m + taskmanager.numberOfTaskSlots: 1 + jobmanager.numberOfTaskSlots: 1 + parallelism.default: 1 + jobmanager.execution.failover-strategy: region + taskmanager.memory.network.fraction: 0.1 + scheduler-mode: reactive + heartbeat.timeout: 8000 + heartbeat.interval: 5000 + state.savepoints.dir: file:///tmp + +commonAnnotations: + reloader.stakater.com/auto: "true" \ No newline at end of file From 02d1c660eab4937e0ba9f07b0b904295c46035ad Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Wed, 7 Aug 2024 11:18:40 +0530 Subject: [PATCH 118/235] #OBS-I108: helm chart fixes --- .../flink/templates/deployment.yaml | 4 +- .../flink/templates/servicemonitor.yaml | 2 +- command-service/helm-charts/flink/values.yaml | 174 +----------------- 3 files changed, 5 insertions(+), 175 deletions(-) diff --git a/command-service/helm-charts/flink/templates/deployment.yaml b/command-service/helm-charts/flink/templates/deployment.yaml index cd7f5619..0a77458d 100644 --- a/command-service/helm-charts/flink/templates/deployment.yaml +++ b/command-service/helm-charts/flink/templates/deployment.yaml @@ -74,7 +74,7 @@ spec: ports: - containerPort: {{ .Values.taskmanager.rpc_port }} name: rpc - {{- toYaml (index $jobData "resources" $component )| nindent 10 }} + {{- toYaml (index .Values.flink_resources $component )| nindent 10 }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: @@ -186,7 +186,7 @@ spec: - name: {{ .name }} containerPort: {{ .targetPort }} {{- end }} - {{- toYaml (index $jobData "resources" $component )| nindent 10 }} + {{- toYaml (index .Values.flink_resources $component )| nindent 10 }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: diff --git a/command-service/helm-charts/flink/templates/servicemonitor.yaml b/command-service/helm-charts/flink/templates/servicemonitor.yaml index 8f7160c9..42132ef9 100644 --- a/command-service/helm-charts/flink/templates/servicemonitor.yaml +++ b/command-service/helm-charts/flink/templates/servicemonitor.yaml @@ -1,6 +1,6 @@ {{- $currentScope := .}} {{- range $jobName, $jobData := .Values.flink_jobs }} -{{- range $key, $smData := $jobData.serviceMonitor }} +{{- range $key, $smData := .Values.serviceMonitor }} {{- if $jobData.enabled}} {{- if $smData.enabled}} {{- with $currentScope }} diff --git a/command-service/helm-charts/flink/values.yaml b/command-service/helm-charts/flink/values.yaml index a7bd26e3..52b5f0f4 100644 --- a/command-service/helm-charts/flink/values.yaml +++ b/command-service/helm-charts/flink/values.yaml @@ -374,7 +374,7 @@ baseconfig: | # limits: # cpu: 1 # memory: 1024Mi -dummy_flink_resource: &flink_resources +flink_resources: taskmanager: resources: requests: @@ -392,7 +392,7 @@ dummy_flink_resource: &flink_resources cpu: 1 memory: 1024Mi -serviceMonitor: &serviceMonitor +serviceMonitor: jobmanager: enabled: true interval: 30s @@ -420,8 +420,6 @@ flink_jobs: imagePullSecrets: [] job_classname: in.sanketika.obsrv.pipeline.task.MasterDataProcessorStreamTask - resources: *flink_resources - serviceMonitor: *serviceMonitor config: |+ include file("/data/flink/conf/baseconfig.conf") @@ -439,173 +437,5 @@ flink_jobs: heartbeat.interval: 5000 state.savepoints.dir: file:///tmp - unified-pipeline: - enabled: false - - registry: sanketikahub - repository: unified-pipeline - tag: 1.0.0-GA - imagePullSecrets: [] - - job_classname: in.sanketika.obsrv.pipeline.task.UnifiedPipelineStreamTask - resources: *flink_resources - serviceMonitor: *serviceMonitor - - config: |+ - include file("/data/flink/conf/baseconfig.conf") - kafka { - input.topic = ${job.env}".ingest" - output.raw.topic = ${job.env}".raw" - output.extractor.duplicate.topic = ${job.env}".failed" - output.batch.failed.topic = ${job.env}".failed" - event.max.size = "1048576" # Max is only 1MB - output.invalid.topic = ${job.env}".failed" - output.unique.topic = ${job.env}".unique" - output.duplicate.topic = ${job.env}".failed" - output.denorm.topic = ${job.env}".denorm" - output.denorm.failed.topic = ${job.env}".failed" - output.transform.topic = ${job.env}".transform" - output.transform.failed.topic = ${job.env}".transform.failed" - stats.topic = ${job.env}".stats" - groupId = ${job.env}"-unified-pipeline-group" - producer { - max-request-size = 5242880 - } - } - - task { - window.time.in.seconds = 5 - window.count = 30 - window.shards = 1400 - consumer.parallelism = 2 - downstream.operators.parallelism = 2 - } - - redis { - database { - extractor.duplication.store.id = 1 - preprocessor.duplication.store.id = 2 - key.expiry.seconds = 3600 - } - } - flink-conf: |+ - jobmanager.memory.flink.size: 1024m - taskmanager.memory.flink.size: 1024m - taskmanager.numberOfTaskSlots: 2 - jobmanager.numberOfTaskSlots: 1 - parallelism.default: 1 - jobmanager.execution.failover-strategy: region - taskmanager.memory.network.fraction: 0.1 - scheduler-mode: reactive - heartbeat.timeout: 8000 - heartbeat.interval: 5000 - state.savepoints.dir: file:///tmp - - transformer-ext: - enabled: false - - registry: sanketikahub - repository: transformer-ext - tag: 1.0.0-GA - imagePullSecrets: [] - - job_classname: in.sanketika.obsrv.transformer.task.TransformerStreamTask - resources: *flink_resources - serviceMonitor: *serviceMonitor - - config: |+ - include file("/data/flink/conf/baseconfig.conf") - kafka { - input.topic = ${job.env}".denorm" - output.transform.topic = ${job.env}".transform" - output.transform.failed.topic = ${job.env}".transform.failed" - groupId = ${job.env}"-transformer-group" - producer { - max-request-size = 5242880 - } - } - - task { - consumer.parallelism = 1 - downstream.operators.parallelism = 1 - } - flink-conf: |+ - jobmanager.memory.flink.size: 1024m - taskmanager.memory.flink.size: 1024m - taskmanager.numberOfTaskSlots: 2 - jobmanager.numberOfTaskSlots: 1 - parallelism.default: 1 - jobmanager.execution.failover-strategy: region - taskmanager.memory.network.fraction: 0.1 - scheduler-mode: reactive - heartbeat.timeout: 8000 - heartbeat.interval: 5000 - state.savepoints.dir: file:///tmp - - master-data-processor-ext: - enabled: false - - registry: sanketikahub - repository: master-data-processor-ext - tag: 1.0.0-GA - imagePullSecrets: [] - - job_classname: in.sanketika.obsrv.pipeline.task.MasterDataProcessorStreamTask - resources: *flink_resources - serviceMonitor: *serviceMonitor - - config: |+ - include file("/data/flink/conf/baseconfig.conf") - kafka { - input.topic = ${job.env}".masterdata.ingest" - output.raw.topic = ${job.env}".masterdata.raw" - output.extractor.duplicate.topic = ${job.env}".masterdata.failed" - output.failed.topic = ${job.env}".masterdata.failed" - output.batch.failed.topic = ${job.env}".masterdata.failed" - event.max.size = "1048576" # Max is only 1MB - output.invalid.topic = ${job.env}".masterdata.failed" - output.unique.topic = ${job.env}".masterdata.unique" - output.duplicate.topic = ${job.env}".masterdata.failed" - output.denorm.topic = ${job.env}".masterdata.denorm" - output.transform.topic = ${job.env}".masterdata.transform" - output.transform.failed.topic = ${job.env}".masterdata.transform.failed" - stats.topic = ${job.env}".masterdata.stats" - groupId = ${job.env}"-masterdata-pipeline-group" - - producer { - max-request-size = 5242880 - } - } - - task { - window.time.in.seconds = 5 - window.count = 30 - window.shards = 1400 - consumer.parallelism = 1 - downstream.operators.parallelism = 1 - } - - redis { - database { - extractor.duplication.store.id = 1 - preprocessor.duplication.store.id = 2 - key.expiry.seconds = 3600 - } - } - - dataset.type = "master-dataset" - flink-conf: |+ - jobmanager.memory.flink.size: 1024m - taskmanager.memory.flink.size: 1024m - taskmanager.numberOfTaskSlots: 1 - jobmanager.numberOfTaskSlots: 1 - parallelism.default: 1 - jobmanager.execution.failover-strategy: region - taskmanager.memory.network.fraction: 0.1 - scheduler-mode: reactive - heartbeat.timeout: 8000 - heartbeat.interval: 5000 - state.savepoints.dir: file:///tmp - commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file From 74ba27552ad0cd86f3eb792564f0a0d9b9a7fa1e Mon Sep 17 00:00:00 2001 From: Anand Parthasarathy Date: Wed, 7 Aug 2024 15:13:06 +0530 Subject: [PATCH 119/235] #OBS-I108: feat: Modify volume mounts --- .../flink/templates/configmap.yaml | 30 ---------- .../flink/templates/deployment.yaml | 58 ++++--------------- .../flink/templates/servicemonitor.yaml | 2 +- command-service/helm-charts/flink/values.yaml | 18 +----- 4 files changed, 14 insertions(+), 94 deletions(-) delete mode 100644 command-service/helm-charts/flink/templates/configmap.yaml diff --git a/command-service/helm-charts/flink/templates/configmap.yaml b/command-service/helm-charts/flink/templates/configmap.yaml deleted file mode 100644 index b486db7f..00000000 --- a/command-service/helm-charts/flink/templates/configmap.yaml +++ /dev/null @@ -1,30 +0,0 @@ -{{- $currentScope := .}} -{{- range $jobName, $jobData := .Values.flink_jobs }} -{{- if $jobData.enabled }} -{{- with $currentScope }} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ printf "%s-config" $jobName }} - namespace: {{ include "base.namespace" . }} - labels: {{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) | nindent 4 }} - {{- if .Values.commonAnnotations }} - annotations: -{{ include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} - {{- end }} -data: - {{- range $configKey, $configValue := $jobData -}} - {{/* We don't have to create key value pair configmaps as files */}} - {{- if contains "\n" (toString $configValue) -}} - {{- $configKey | nindent 2 -}}: |- - {{- include "common.tplvalues.render" (dict "value" $configValue "context" $) | nindent 4 }} - {{- end -}} - {{- end -}} - {{- "baseconfig" | nindent 2 -}}: |- - {{- include "common.tplvalues.render" (dict "value" .Values.baseconfig "context" $) | nindent 4 }} - {{- "log4j_console_properties" | nindent 2 -}}: |- - {{- include "common.tplvalues.render" (dict "value" .Values.log4j_console_properties "context" $) | nindent 4 }} -{{- end -}} -{{- end -}} -{{- end -}} diff --git a/command-service/helm-charts/flink/templates/deployment.yaml b/command-service/helm-charts/flink/templates/deployment.yaml index 0a77458d..4487bb47 100644 --- a/command-service/helm-charts/flink/templates/deployment.yaml +++ b/command-service/helm-charts/flink/templates/deployment.yaml @@ -79,30 +79,15 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - name: flink-config-volume - mountPath: /opt/flink/conf/flink-conf.yaml - subPath: flink-conf.yaml - - name: flink-config-volume - mountPath: /data/flink/conf/baseconfig.conf - subPath: baseconfig.conf - - name: flink-config-volume - mountPath: /data/flink/conf/{{ $jobName }}.conf - subPath: {{ $jobName }}.conf - - name: flink-config-volume - mountPath: /opt/flink/conf/log4j-console.properties - subPath: log4j-console.properties + mountPath: /data/flink/conf/flink-connector.conf + subPath: flink-connector.conf volumes: - name: flink-config-volume configMap: - name: {{ $jobName }}-config + name: flink-connector-conf items: - - key: flink-conf - path: flink-conf.yaml - - key: baseconfig - path: baseconfig.conf - - key: config - path: {{ $jobName }}.conf - - key: log4j_console_properties - path: log4j-console.properties + - key: connectors-scala-config.conf + path: flink-connector.conf {{ $component := "jobmanager" }} --- apiVersion: apps/v1 @@ -153,11 +138,7 @@ spec: command: - "/bin/sh" - "-c" - - "/usr/bin/python3.11 -c 'print(5**3)'" - - "/bin/sh" - - "-c" - - "/opt/flink/bin/standalone-job.sh" - args: ["start-foreground", + args: ["/usr/bin/python3.11 /data/flink/connectors/connectors-init/connector.py", "/opt/flink/bin/standalone-job.sh start-foreground", "--job-classname={{ index $jobData "job_classname" }}", {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}", @@ -180,7 +161,7 @@ spec: "-Dblob.server.port={{ .Values.jobmanager.blob_port }}", "-Dqueryable-state.server.ports={{ .Values.jobmanager.query_port }}", "--config.file.path", - "/data/flink/conf/{{ $jobName }}.conf"] + "/data/flink/conf/flink-connector.conf"] ports: {{- range .Values.service.ports }} - name: {{ .name }} @@ -191,30 +172,15 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - name: flink-config-volume - mountPath: /opt/flink/conf/flink-conf.yaml - subPath: flink-conf.yaml - - name: flink-config-volume - mountPath: /data/flink/conf/baseconfig.conf - subPath: baseconfig.conf - - name: flink-config-volume - mountPath: /data/flink/conf/{{ $jobName }}.conf - subPath: {{ $jobName }}.conf - - name: flink-config-volume - mountPath: /opt/flink/conf/log4j-console.properties - subPath: log4j-console.properties + mountPath: /data/flink/conf/flink-connector.conf + subPath: flink-connector.conf volumes: - name: flink-config-volume configMap: - name: {{ $jobName }}-config + name: flink-connector-conf items: - - key: flink-conf - path: flink-conf.yaml - - key: baseconfig - path: baseconfig.conf - - key: config - path: {{ $jobName }}.conf - - key: log4j_console_properties - path: log4j-console.properties + - key: connectors-scala-config.conf + path: flink-connector.conf {{- end }} {{- end}} {{- end}} \ No newline at end of file diff --git a/command-service/helm-charts/flink/templates/servicemonitor.yaml b/command-service/helm-charts/flink/templates/servicemonitor.yaml index 42132ef9..e934bda0 100644 --- a/command-service/helm-charts/flink/templates/servicemonitor.yaml +++ b/command-service/helm-charts/flink/templates/servicemonitor.yaml @@ -1,7 +1,7 @@ {{- $currentScope := .}} {{- range $jobName, $jobData := .Values.flink_jobs }} -{{- range $key, $smData := .Values.serviceMonitor }} {{- if $jobData.enabled}} +{{- range $key, $smData := $.Values.serviceMonitor }} {{- if $smData.enabled}} {{- with $currentScope }} --- diff --git a/command-service/helm-charts/flink/values.yaml b/command-service/helm-charts/flink/values.yaml index 52b5f0f4..ccaead68 100644 --- a/command-service/helm-charts/flink/values.yaml +++ b/command-service/helm-charts/flink/values.yaml @@ -419,23 +419,7 @@ flink_jobs: tag: 1.0.0 imagePullSecrets: [] - job_classname: in.sanketika.obsrv.pipeline.task.MasterDataProcessorStreamTask - - config: |+ - include file("/data/flink/conf/baseconfig.conf") - - flink-conf: |+ - jobmanager.memory.flink.size: 1024m - taskmanager.memory.flink.size: 1024m - taskmanager.numberOfTaskSlots: 2 - jobmanager.numberOfTaskSlots: 1 - parallelism.default: 1 - jobmanager.execution.failover-strategy: region - taskmanager.memory.network.fraction: 0.1 - scheduler-mode: reactive - heartbeat.timeout: 8000 - heartbeat.interval: 5000 - state.savepoints.dir: file:///tmp + job_classname: org.sunbird.obsrv.connector.KafkaConnector commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file From e7059935660b058cf2ce6fdb79edef132cd57841 Mon Sep 17 00:00:00 2001 From: Anand Parthasarathy Date: Wed, 7 Aug 2024 16:29:13 +0530 Subject: [PATCH 120/235] #OBS-I108: feat: Add PVC for JobManager --- .../flink/templates/deployment.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/command-service/helm-charts/flink/templates/deployment.yaml b/command-service/helm-charts/flink/templates/deployment.yaml index 4487bb47..77beaf96 100644 --- a/command-service/helm-charts/flink/templates/deployment.yaml +++ b/command-service/helm-charts/flink/templates/deployment.yaml @@ -49,7 +49,7 @@ spec: containers: - name: {{ $jobName }}-taskmanager image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "IfNotPresent" }} + imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} workingDir: {{ .Values.taskmanager.flink_work_dir }} command: ["/opt/flink/bin/taskmanager.sh"] args: ["start-foreground", @@ -91,7 +91,7 @@ spec: {{ $component := "jobmanager" }} --- apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: name: {{ printf "%s-%s" $jobName $component }} namespace: {{ include "base.namespace" . }} @@ -174,6 +174,8 @@ spec: - name: flink-config-volume mountPath: /data/flink/conf/flink-connector.conf subPath: flink-connector.conf + - name: data + mountPath: /flink/connectors volumes: - name: flink-config-volume configMap: @@ -181,6 +183,18 @@ spec: items: - key: connectors-scala-config.conf path: flink-connector.conf + + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + storageClassName: "standard" + {{- end }} {{- end}} {{- end}} \ No newline at end of file From a6ba601f710018afcb0fde9618a5948b47bef0df Mon Sep 17 00:00:00 2001 From: Anand Parthasarathy Date: Wed, 7 Aug 2024 18:15:09 +0530 Subject: [PATCH 121/235] #OBS-I108: feat: change args for jobmanager command --- .../flink/templates/deployment.yaml | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/command-service/helm-charts/flink/templates/deployment.yaml b/command-service/helm-charts/flink/templates/deployment.yaml index 77beaf96..31c75b96 100644 --- a/command-service/helm-charts/flink/templates/deployment.yaml +++ b/command-service/helm-charts/flink/templates/deployment.yaml @@ -135,33 +135,34 @@ spec: image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} workingDir: /opt/flink - command: - - "/bin/sh" - - "-c" - args: ["/usr/bin/python3.11 /data/flink/connectors/connectors-init/connector.py", "/opt/flink/bin/standalone-job.sh start-foreground", - "--job-classname={{ index $jobData "job_classname" }}", - {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}", + command: ["/bin/sh", "-c"] + args: + - | + /usr/bin/python3.11 /data/flink/connectors/connectors-init/connector.py + exec /opt/flink/bin/standalone-job.sh start-foreground \ + "--job-classname={{ index $jobData "job_classname" }}" \ + {{- if eq .Values.checkpoint_store_type "azure" }} + "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ {{- end }} {{- if and (eq .Values.checkpoint_store_type "s3") (ne .Values.s3_auth_type "serviceAccount") }} - "-Ds3.access-key={{ .Values.s3_access_key }}", - "-Ds3.secret-key={{ .Values.s3_secret_key }}", - "-Ds3.endpoint={{ .Values.s3_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", + "-Ds3.access-key={{ .Values.s3_access_key }}" \ + "-Ds3.secret-key={{ .Values.s3_secret_key }}" \ + "-Ds3.endpoint={{ .Values.s3_endpoint }}" \ + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}" \ {{- end }} {{- if eq .Values.checkpoint_store_type "gcs" }} - "-Dgoogle.cloud.auth.service.account.enable=true", + "-Dgoogle.cloud.auth.service.account.enable=true" \ {{- end }} - "-Dweb.submit.enable=false", - "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", - "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}", - "-Djobmanager.rpc.address={{ $jobName }}-jobmanager", - "-Djobmanager.rpc.port={{ .Values.jobmanager.rpc_port }}", - "-Dparallelism.default=1", - "-Dblob.server.port={{ .Values.jobmanager.blob_port }}", - "-Dqueryable-state.server.ports={{ .Values.jobmanager.query_port }}", - "--config.file.path", - "/data/flink/conf/flink-connector.conf"] + "-Dweb.submit.enable=false" \ + "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter" \ + "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}" \ + "-Djobmanager.rpc.address={{ $jobName }}-jobmanager" \ + "-Djobmanager.rpc.port={{ .Values.jobmanager.rpc_port }}" \ + "-Dparallelism.default=1" \ + "-Dblob.server.port={{ .Values.jobmanager.blob_port }}" \ + "-Dqueryable-state.server.ports={{ .Values.jobmanager.query_port }}" \ + "--config.file.path" \ + "/data/flink/conf/flink-connector.conf" ports: {{- range .Values.service.ports }} - name: {{ .name }} From f0e53e149620c2845a99b46ecae0bbb45ff56d4f Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:33:06 +0530 Subject: [PATCH 122/235] #OBS-I141: added a new metric to sum the response time --- api-service/src/metrics/prometheus/helpers.ts | 17 +++++++++++++---- api-service/src/metrics/prometheus/index.ts | 7 ++++--- api-service/src/metrics/prometheus/metrics.ts | 10 +++++++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 5e0442c1..13d7f9d7 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -1,5 +1,5 @@ import { NextFunction, Response } from "express"; -import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime } from "."; +import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; @@ -19,7 +19,10 @@ export const onSuccess = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 200 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementSuccessfulApiCalls({ labels }) } @@ -28,7 +31,10 @@ export const onFailure = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 500 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } @@ -37,7 +43,10 @@ export const onGone = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 410 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } diff --git a/api-service/src/metrics/prometheus/index.ts b/api-service/src/metrics/prometheus/index.ts index da143f9f..3fdefe19 100644 --- a/api-service/src/metrics/prometheus/index.ts +++ b/api-service/src/metrics/prometheus/index.ts @@ -1,6 +1,6 @@ import client from "prom-client"; -import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric } from "./metrics" -const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric]; +import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric } from "./metrics" +const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric]; import { NextFunction } from "express"; const register = new client.Registry(); @@ -17,6 +17,7 @@ const incrementApiCalls = ({ labels = {} }: Record) => totalApiCall const setQueryResponseTime = ({ labels = {}, duration }: Record) => queryResponseTimeMetric.labels(labels).set(duration); const incrementFailedApiCalls = ({ labels = {} }: Record) => failedApiCallsMetric.labels(labels).inc(); const incrementSuccessfulApiCalls = ({ labels = {} }: Record) => successApiCallsMetric.labels(labels).inc(); +const incrementResponseTime = ({ labels = {}, duration }: Record) => sumResponseTimeMetric.labels(labels).inc(duration); //register the metrics configureRegistry(register); @@ -31,5 +32,5 @@ const metricsScrapeHandler = async (req: any, res: any, next: NextFunction) => { } } -export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls }; +export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls, incrementResponseTime}; diff --git a/api-service/src/metrics/prometheus/metrics.ts b/api-service/src/metrics/prometheus/metrics.ts index 2985dc0f..499bd15a 100644 --- a/api-service/src/metrics/prometheus/metrics.ts +++ b/api-service/src/metrics/prometheus/metrics.ts @@ -28,9 +28,17 @@ const successApiCallsMetric = new Prometheus.Counter({ labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] }) +// Create a new Prometheus Counter for sum of response time +const sumResponseTimeMetric = new Prometheus.Counter({ + name: "node_sum_response_time", + help: "The sum of response time for time series of same label", + labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] +}); + export { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, - successApiCallsMetric + successApiCallsMetric, + sumResponseTimeMetric } \ No newline at end of file From de6da1d2920ae59178df29f2709fb265071b99bc Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:34:09 +0530 Subject: [PATCH 123/235] #OBS-I141: modified the url variable and access dataset_id from params --- api-service/src/metrics/prometheus/helpers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 13d7f9d7..b87df8a7 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -52,12 +52,12 @@ export const onGone = (req: any, res: Response) => { } const getMetricLabels = (req: any, res: Response) => { - const { id, entity, url, startTime } = req; + const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res const request_size = req.socket.bytesRead const response_size = res.getHeader("content-length"); - const dataset_id = _.get(req, "dataset_id") || null + const dataset_id = _.get(req, ["body", "request", "dataset_id"]) || _.get(req, ["params", "dataset_id"]) || null const duration = getDuration(startTime); - const metricLabels = { entity, id, endpoint: url, dataset_id, status: statusCode, request_size, response_size } + const metricLabels = { entity, id, endpoint: originalUrl, dataset_id, status: statusCode, request_size, response_size } return { duration, metricLabels } } From ea7b0f85ec0947c180b795d9f54ac67fd3751029 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:35:45 +0530 Subject: [PATCH 124/235] #OBS-I141: added helper function to get dataset_id for error cases --- api-service/src/helpers/ResponseHandler.ts | 4 ++-- api-service/src/metrics/prometheus/helpers.ts | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/api-service/src/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts index 3bac60b7..e51eef27 100644 --- a/api-service/src/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -1,7 +1,7 @@ import { NextFunction, Request, Response } from "express"; import httpStatus from "http-status"; import { IResponse, Result } from "../types/DatasetModels"; -import { onFailure, onSuccess } from "../metrics/prometheus/helpers"; +import { onFailure, onObsrvFailure, onSuccess } from "../metrics/prometheus/helpers"; import moment from "moment"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; @@ -42,7 +42,7 @@ const ResponseHandler = { const resmsgid = _.get(res, "resmsgid") const response = ResponseHandler.refactorResponse({ id, msgid, params: { status: "FAILED" }, responseCode: errCode || httpStatus["500_NAME"], resmsgid, result: data }) res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json({ ...response, error: { code, message } }); - entity && onFailure(req, res) + entity && onObsrvFailure(req,res,error) }, setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index b87df8a7..2f502d99 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -2,6 +2,7 @@ import { NextFunction, Response } from "express"; import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; +import { ObsrvError } from "../../types/ObsrvError"; export const onRequest = ({ entity = Entity.Management }: any) => (req: any, res: Response, next: NextFunction) => { const startTime = Date.now(); @@ -51,6 +52,19 @@ export const onGone = (req: any, res: Response) => { incrementFailedApiCalls({ labels }); } +export const onObsrvFailure = (req: any, res: Response,error: ObsrvError) => { + const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) + metricLabels.dataset_id = error.datasetId + const { statusCode = 404 } = res + const labels = { ...metricLabels, status: statusCode } + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } + incrementApiCalls({ labels }) + incrementFailedApiCalls({ labels }); +} + const getMetricLabels = (req: any, res: Response) => { const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res From fbdcf8a4603000be066c53b470e0eb07174f85f0 Mon Sep 17 00:00:00 2001 From: Anand Parthasarathy Date: Fri, 9 Aug 2024 12:38:38 +0530 Subject: [PATCH 125/235] #OBS-I108: feat: Use sidecar container to submit connector flink job --- .../flink/templates/configmap.yaml | 21 ++++ .../flink/templates/deployment.yaml | 103 ++++++++++++++---- command-service/helm-charts/flink/values.yaml | 90 --------------- 3 files changed, 102 insertions(+), 112 deletions(-) create mode 100644 command-service/helm-charts/flink/templates/configmap.yaml diff --git a/command-service/helm-charts/flink/templates/configmap.yaml b/command-service/helm-charts/flink/templates/configmap.yaml new file mode 100644 index 00000000..3a90fa01 --- /dev/null +++ b/command-service/helm-charts/flink/templates/configmap.yaml @@ -0,0 +1,21 @@ +{{- $currentScope := .}} +{{- range $jobName, $jobData := .Values.flink_jobs }} +{{- if $jobData.enabled }} +{{- with $currentScope }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-config" $jobName }} + namespace: {{ include "base.namespace" . }} + labels: {{ include "common.labels.standard" (dict "customLabels" .Values.commonLabels "context" $) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: +{{ include "common.tplvalues.render" (dict "value" .Values.commonAnnotations "context" $) | nindent 4 }} + {{- end }} +data: + {{- "log4j_console_properties" | nindent 2 -}}: |- + {{- include "common.tplvalues.render" (dict "value" .Values.log4j_console_properties "context" $) | nindent 4 }} +{{- end -}} +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/command-service/helm-charts/flink/templates/deployment.yaml b/command-service/helm-charts/flink/templates/deployment.yaml index 31c75b96..1cd613b3 100644 --- a/command-service/helm-charts/flink/templates/deployment.yaml +++ b/command-service/helm-charts/flink/templates/deployment.yaml @@ -51,6 +51,17 @@ spec: image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} workingDir: {{ .Values.taskmanager.flink_work_dir }} + # args: ["taskmanager"] + # env: + # - name: FLINK_PROPERTIES + # value: |+ + # jobmanager.rpc.address: {{ $jobName }}-jobmanager + # taskmanager.rpc.port=6122 + # taskmanager.numberOfTaskSlots: 2 + # metrics.reporters: prom + # metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory + # metrics.reporter.prom.host: {{ $jobName }}-taskmanager + # metrics.reporter.prom.port: 9251 command: ["/opt/flink/bin/taskmanager.sh"] args: ["start-foreground", {{- if eq .Values.checkpoint_store_type "azure" }} @@ -81,6 +92,12 @@ spec: - name: flink-config-volume mountPath: /data/flink/conf/flink-connector.conf subPath: flink-connector.conf + # - name: flink-config-volume + # mountPath: /opt/flink/conf/flink-conf.yaml + # subPath: flink-conf.yaml + - name: flink-common-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties volumes: - name: flink-config-volume configMap: @@ -88,6 +105,14 @@ spec: items: - key: connectors-scala-config.conf path: flink-connector.conf + - name: flink-common-volume + configMap: + name: {{ $jobName }}-config + items: + # - key: flink-conf + # path: flink-conf.yaml + - key: log4j_console_properties + path: log4j-console.properties {{ $component := "jobmanager" }} --- apiVersion: apps/v1 @@ -135,34 +160,55 @@ spec: image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} workingDir: /opt/flink - command: ["/bin/sh", "-c"] - args: + # command: ["/bin/sh", "-c"] + args: ["jobmanager"] + env: + - name: FLINK_PROPERTIES + value: |+ + jobmanager.rpc.address: {{ $jobName }}-jobmanager + jobmanager.rpc.port=6123 + metrics.reporters: prom + metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory + metrics.reporter.prom.host: {{ $jobName }}-jobmanager + metrics.reporter.prom.port: 9250 + + volumeMounts: + - name: flink-config-volume + mountPath: /data/flink/conf/flink-connector.conf + subPath: flink-connector.conf + - name: data + mountPath: /flink/connectors + - name: flink-common-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties + + - name: {{ $jobName }}-job-submit + image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} + imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + workingDir: /opt/flink + command: + - /bin/bash + - -c - | - /usr/bin/python3.11 /data/flink/connectors/connectors-init/connector.py - exec /opt/flink/bin/standalone-job.sh start-foreground \ - "--job-classname={{ index $jobData "job_classname" }}" \ - {{- if eq .Values.checkpoint_store_type "azure" }} + /usr/bin/python3.11 /data/flink/connectors/connectors-init/connector.py; + sleep 30s; + /opt/flink/bin/flink run -m \ + {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ + /flink/connectors/{{ $jobName }}-1.0.0/{{ $jobName }}-1.0.0.jar \ + --config.file.path /data/flink/conf/flink-connector.conf \ + {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ - {{- end }} - {{- if and (eq .Values.checkpoint_store_type "s3") (ne .Values.s3_auth_type "serviceAccount") }} + {{- end }} + {{- if and (eq .Values.checkpoint_store_type "s3") (ne .Values.s3_auth_type "serviceAccount") }} "-Ds3.access-key={{ .Values.s3_access_key }}" \ "-Ds3.secret-key={{ .Values.s3_secret_key }}" \ "-Ds3.endpoint={{ .Values.s3_endpoint }}" \ "-Ds3.path.style.access={{ .Values.s3_path_style_access }}" \ - {{- end }} - {{- if eq .Values.checkpoint_store_type "gcs" }} - "-Dgoogle.cloud.auth.service.account.enable=true" \ - {{- end }} - "-Dweb.submit.enable=false" \ - "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter" \ - "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}" \ - "-Djobmanager.rpc.address={{ $jobName }}-jobmanager" \ - "-Djobmanager.rpc.port={{ .Values.jobmanager.rpc_port }}" \ - "-Dparallelism.default=1" \ - "-Dblob.server.port={{ .Values.jobmanager.blob_port }}" \ - "-Dqueryable-state.server.ports={{ .Values.jobmanager.query_port }}" \ - "--config.file.path" \ - "/data/flink/conf/flink-connector.conf" + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcs" }} + "-Dgoogle.cloud.auth.service.account.enable=true" \ + {{- end }} + ; ports: {{- range .Values.service.ports }} - name: {{ .name }} @@ -177,6 +223,11 @@ spec: subPath: flink-connector.conf - name: data mountPath: /flink/connectors + - name: flink-common-volume + mountPath: /opt/flink/conf/log4j-console.properties + subPath: log4j-console.properties + + volumes: - name: flink-config-volume configMap: @@ -184,6 +235,14 @@ spec: items: - key: connectors-scala-config.conf path: flink-connector.conf + - name: flink-common-volume + configMap: + name: {{ $jobName }}-config + items: + # - key: flink-conf + # path: flink-conf.yaml + - key: log4j_console_properties + path: log4j-console.properties volumeClaimTemplates: - metadata: diff --git a/command-service/helm-charts/flink/values.yaml b/command-service/helm-charts/flink/values.yaml index ccaead68..f0ea188c 100644 --- a/command-service/helm-charts/flink/values.yaml +++ b/command-service/helm-charts/flink/values.yaml @@ -149,95 +149,6 @@ cloud_storage_content_bucketname: sunbird-content-dev cert_container_name: dev-e-credentials # cloud_storage_endpoint: https://{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net -flink_dp_storage_container: "" -checkpoint_interval: 60000 -checkpoint_pause_between_seconds: 5000 -checkpoint_compression_enabled: true -restart_attempts: 3 -restart_delay: 30000 # in milli-seconds -producer_max_request_size: 1572864 -producer_batch_size: 98304 -producer_linger_ms: 10 -producer_compression: snappy -extractor_event_max_size: 1048576 # Max is only 1MB -extractor_max_request_size: 5242880 -extractor_consumer_parallelism: 1 -extractor_operators_parallelism: 1 -telemetry_extractor_key_expiry_seconds: 3600 -ingest_router_consumer_parallelism: 1 -ingest_router_operators_parallelism: 1 -raw_router_consumer_parallelism: 1 -raw_router_downstream_parallelism: 1 - -content_port: 6379 -device_port: 6380 -user_port: 6381 -dialcode_port: 6382 - -pipeline_preprocessor_consumer_parallelism: 1 -pipeline_preprocessor_operators_parallelism: 1 -portal_id: ".sunbird.portal" -desktop_id: ".sunbird.desktop" -pipeline_preprocessor_key_expiry_seconds: 3600 - -denorm_secondary_window_count: 30 -denorm_secondary_window_shards: 1400 -denorm_primary_window_count: 30 -denorm_primary_window_shards: 1400 - -denorm_summary_window_count: 5 -denorm_summary_window_shards: 1400 - -denorm_secondary_consumer_parallelism: 1 -telemetry_denorm_secondary_operators_parallelism: 1 -denorm_primary_consumer_parallelism: 1 -telemetry_denorm_primary_operators_parallelism: 1 - -### summary-denormalization related vars -summary_denorm_consumer_parallelism: 1 -summary_denorm_operators_parallelism: 1 -summary_denorm_duplication_key_expiry_seconds: 3600 -summary_denorm_key_expiry_seconds: 3600 - -### De-normalization related vars -denorm_consumer_parallelism: 1 -telemetry_denorm_operators_parallelism: 1 -de_normalization_duplicationstore_key_expiry_seconds: 3600 -de_normalization_key_expiry_seconds: 3600 -denorm_window_count: 30 -denorm_window_shards: 1400 - -### Druid-validator related vars -druid_validator_consumer_parallelism: 1 -druid_validator_operators_parallelism: 1 -druid_validator_key_expiry_seconds: 3600 -druid_validation_enabled: true -druid_deduplication_enabled: true - -### error-denormalization related vars -error_denorm_consumer_parallelism: 1 -error_denorm_operators_parallelism: 1 - -### Device-profile-updater related vars -deviceprofile_parallelism: 1 -device_profile_updater_key_expiry_seconds: 3600 -device_profile_table: "_device_profile" - -### content-cache-updater -# dialcode_api_url: {{ .Values.global.proto }}://{{ .Values.global.domain_name }}/{{ .Values.dialcode_endpoint }} -dialcode_api_auth_key: "" - -#(user read api details) -user_read_api_endpoint: "/private/user/v1/read/" -user_read_api_url: "learner-service:9000" - -### User-cache-updater related vars -usercache_updater_parallelism: 1 -user_cache_updater_key_expiry_seconds: 3600 -middleware_cassandra_keyspace: sunbird -middleware_cassandra_user_table: user -middleware_cassandra_location_table: location - postgres: max_connections: 2 sslmode: false @@ -413,7 +324,6 @@ serviceMonitor: flink_jobs: kafka-connector: enabled: true - registry: surabhi1510 repository: flink-python-3.11 tag: 1.0.0 From bc69d4b805f672cbddb99af9beb6820458837075 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:44:08 +0530 Subject: [PATCH 126/235] #OBS-I141: added telemetry for v2 api's --- api-service/src/app.ts | 3 +++ api-service/src/routes/Router.ts | 26 ++++++++++--------- api-service/src/services/telemetry.ts | 4 +-- api-service/src/telemetry/telemetryActions.ts | 5 +++- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index ec2a10cd..962331e9 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -8,13 +8,16 @@ import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; import { alertsRouter } from "./routes/AlertsRouter"; +import { interceptAuditEvents } from "./services/telemetry"; const app: Application = express(); + app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.use(errorHandler) +app.use(interceptAuditEvents()); app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 3f2d623a..444a318d 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -27,16 +27,18 @@ import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; +import {OperationType, telemetryAuditStart} from "../services/telemetry"; +import telemetryActions from "../telemetry/telemetryActions"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), dataIn); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), dataIn); router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), dataExhaust); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), dataExhaust); router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); @@ -45,14 +47,14 @@ router.patch("/template/update/:templateId", setDataToRequestObject("api.query.t router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); +router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); +router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), ConnectorsRead); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); //Wrapper Service diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index ede157f7..e6139d94 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -4,7 +4,7 @@ import _ from "lodash"; import { config as appConfig } from "../configs/Config"; import { Kafka } from "kafkajs"; -const env = _.get(appConfig, "env") +const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); const brokerServers = _.get(appConfig, "telemetry_service_config.kafka.config.brokers"); @@ -29,7 +29,7 @@ const getDefaults = () => { sid: v4(), pdata: { id: `${env}.api.service`, - ver: "1.0.0" + ver: `${version}` } }, object: {}, diff --git a/api-service/src/telemetry/telemetryActions.ts b/api-service/src/telemetry/telemetryActions.ts index bbee6faa..befe6100 100644 --- a/api-service/src/telemetry/telemetryActions.ts +++ b/api-service/src/telemetry/telemetryActions.ts @@ -17,5 +17,8 @@ export default { "sqlQuery": "dataset:query:sql", "ingestEvents": "dataset:events:ingest", "submitIngestionSpec": "datasource:ingestion:submit", - "datasetExhaust": "dataset:exhaust:get" + "datasetExhaust": "dataset:exhaust:get", + "copyDataset": "dataset:copy", + "readConnectors": "connectors:read", + "listConnectors": "connectors:list", } \ No newline at end of file From c3e169737a3fb51d70edbb155c08523cf548dbb5 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:33:06 +0530 Subject: [PATCH 127/235] #OBS-I141: added a new metric to sum the response time --- api-service/src/metrics/prometheus/helpers.ts | 17 +++++++++++++---- api-service/src/metrics/prometheus/index.ts | 7 ++++--- api-service/src/metrics/prometheus/metrics.ts | 10 +++++++++- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 5e0442c1..13d7f9d7 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -1,5 +1,5 @@ import { NextFunction, Response } from "express"; -import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime } from "."; +import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; @@ -19,7 +19,10 @@ export const onSuccess = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 200 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementSuccessfulApiCalls({ labels }) } @@ -28,7 +31,10 @@ export const onFailure = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 500 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } @@ -37,7 +43,10 @@ export const onGone = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 410 } = res const labels = { ...metricLabels, status: statusCode } - duration && setQueryResponseTime({ duration, labels }); + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } diff --git a/api-service/src/metrics/prometheus/index.ts b/api-service/src/metrics/prometheus/index.ts index da143f9f..3fdefe19 100644 --- a/api-service/src/metrics/prometheus/index.ts +++ b/api-service/src/metrics/prometheus/index.ts @@ -1,6 +1,6 @@ import client from "prom-client"; -import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric } from "./metrics" -const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric]; +import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric } from "./metrics" +const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric]; import { NextFunction } from "express"; const register = new client.Registry(); @@ -17,6 +17,7 @@ const incrementApiCalls = ({ labels = {} }: Record) => totalApiCall const setQueryResponseTime = ({ labels = {}, duration }: Record) => queryResponseTimeMetric.labels(labels).set(duration); const incrementFailedApiCalls = ({ labels = {} }: Record) => failedApiCallsMetric.labels(labels).inc(); const incrementSuccessfulApiCalls = ({ labels = {} }: Record) => successApiCallsMetric.labels(labels).inc(); +const incrementResponseTime = ({ labels = {}, duration }: Record) => sumResponseTimeMetric.labels(labels).inc(duration); //register the metrics configureRegistry(register); @@ -31,5 +32,5 @@ const metricsScrapeHandler = async (req: any, res: any, next: NextFunction) => { } } -export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls }; +export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls, incrementResponseTime}; diff --git a/api-service/src/metrics/prometheus/metrics.ts b/api-service/src/metrics/prometheus/metrics.ts index 2985dc0f..499bd15a 100644 --- a/api-service/src/metrics/prometheus/metrics.ts +++ b/api-service/src/metrics/prometheus/metrics.ts @@ -28,9 +28,17 @@ const successApiCallsMetric = new Prometheus.Counter({ labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] }) +// Create a new Prometheus Counter for sum of response time +const sumResponseTimeMetric = new Prometheus.Counter({ + name: "node_sum_response_time", + help: "The sum of response time for time series of same label", + labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] +}); + export { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, - successApiCallsMetric + successApiCallsMetric, + sumResponseTimeMetric } \ No newline at end of file From 195aaab864de2a0a99461c6f4f31f8ac13ff1c6a Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:34:09 +0530 Subject: [PATCH 128/235] #OBS-I141: modified the url variable and access dataset_id from params --- api-service/src/metrics/prometheus/helpers.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 13d7f9d7..b87df8a7 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -52,12 +52,12 @@ export const onGone = (req: any, res: Response) => { } const getMetricLabels = (req: any, res: Response) => { - const { id, entity, url, startTime } = req; + const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res const request_size = req.socket.bytesRead const response_size = res.getHeader("content-length"); - const dataset_id = _.get(req, "dataset_id") || null + const dataset_id = _.get(req, ["body", "request", "dataset_id"]) || _.get(req, ["params", "dataset_id"]) || null const duration = getDuration(startTime); - const metricLabels = { entity, id, endpoint: url, dataset_id, status: statusCode, request_size, response_size } + const metricLabels = { entity, id, endpoint: originalUrl, dataset_id, status: statusCode, request_size, response_size } return { duration, metricLabels } } From 7628bbd3b39fc29bc0144707d6acd6a7f4ee2f88 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:35:45 +0530 Subject: [PATCH 129/235] #OBS-I141: added helper function to get dataset_id for error cases --- api-service/src/helpers/ResponseHandler.ts | 4 ++-- api-service/src/metrics/prometheus/helpers.ts | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/api-service/src/helpers/ResponseHandler.ts b/api-service/src/helpers/ResponseHandler.ts index 3bac60b7..e51eef27 100644 --- a/api-service/src/helpers/ResponseHandler.ts +++ b/api-service/src/helpers/ResponseHandler.ts @@ -1,7 +1,7 @@ import { NextFunction, Request, Response } from "express"; import httpStatus from "http-status"; import { IResponse, Result } from "../types/DatasetModels"; -import { onFailure, onSuccess } from "../metrics/prometheus/helpers"; +import { onFailure, onObsrvFailure, onSuccess } from "../metrics/prometheus/helpers"; import moment from "moment"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; @@ -42,7 +42,7 @@ const ResponseHandler = { const resmsgid = _.get(res, "resmsgid") const response = ResponseHandler.refactorResponse({ id, msgid, params: { status: "FAILED" }, responseCode: errCode || httpStatus["500_NAME"], resmsgid, result: data }) res.status(statusCode || httpStatus.INTERNAL_SERVER_ERROR).json({ ...response, error: { code, message } }); - entity && onFailure(req, res) + entity && onObsrvFailure(req,res,error) }, setApiId: (id: string) => (req: Request, res: Response, next: NextFunction) => { diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index b87df8a7..2f502d99 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -2,6 +2,7 @@ import { NextFunction, Response } from "express"; import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; +import { ObsrvError } from "../../types/ObsrvError"; export const onRequest = ({ entity = Entity.Management }: any) => (req: any, res: Response, next: NextFunction) => { const startTime = Date.now(); @@ -51,6 +52,19 @@ export const onGone = (req: any, res: Response) => { incrementFailedApiCalls({ labels }); } +export const onObsrvFailure = (req: any, res: Response,error: ObsrvError) => { + const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) + metricLabels.dataset_id = error.datasetId + const { statusCode = 404 } = res + const labels = { ...metricLabels, status: statusCode } + if(duration){ + setQueryResponseTime({ duration, labels }) + incrementResponseTime({duration, labels}) + } + incrementApiCalls({ labels }) + incrementFailedApiCalls({ labels }); +} + const getMetricLabels = (req: any, res: Response) => { const { id, entity, originalUrl, startTime } = req; const { statusCode = 200 } = res From 2be8195a2a901e491c2a1d5b6770e9ca25efd2e4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 12:44:08 +0530 Subject: [PATCH 130/235] #OBS-I141: added telemetry for v2 api's --- api-service/src/app.ts | 3 +++ api-service/src/routes/Router.ts | 26 ++++++++++--------- api-service/src/services/telemetry.ts | 4 +-- api-service/src/telemetry/telemetryActions.ts | 5 +++- 4 files changed, 23 insertions(+), 15 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index ec2a10cd..962331e9 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -8,13 +8,16 @@ import { errorHandler, obsrvErrorHandler } from "./middlewares/errors"; import { ResponseHandler } from "./helpers/ResponseHandler"; import { config } from "./configs/Config"; import { alertsRouter } from "./routes/AlertsRouter"; +import { interceptAuditEvents } from "./services/telemetry"; const app: Application = express(); + app.use(bodyParser.json({ limit: config.body_parser_limit})); app.use(express.text()); app.use(express.json()); app.use(errorHandler) +app.use(interceptAuditEvents()); app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 3f2d623a..444a318d 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -27,16 +27,18 @@ import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; +import {OperationType, telemetryAuditStart} from "../services/telemetry"; +import telemetryActions from "../telemetry/telemetryActions"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), dataIn); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), dataIn); router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), dataExhaust); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), dataExhaust); router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); @@ -45,14 +47,14 @@ router.patch("/template/update/:templateId", setDataToRequestObject("api.query.t router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); +router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); +router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), ConnectorsRead); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); //Wrapper Service diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index ede157f7..e6139d94 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -4,7 +4,7 @@ import _ from "lodash"; import { config as appConfig } from "../configs/Config"; import { Kafka } from "kafkajs"; -const env = _.get(appConfig, "env") +const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); const brokerServers = _.get(appConfig, "telemetry_service_config.kafka.config.brokers"); @@ -29,7 +29,7 @@ const getDefaults = () => { sid: v4(), pdata: { id: `${env}.api.service`, - ver: "1.0.0" + ver: `${version}` } }, object: {}, diff --git a/api-service/src/telemetry/telemetryActions.ts b/api-service/src/telemetry/telemetryActions.ts index bbee6faa..befe6100 100644 --- a/api-service/src/telemetry/telemetryActions.ts +++ b/api-service/src/telemetry/telemetryActions.ts @@ -17,5 +17,8 @@ export default { "sqlQuery": "dataset:query:sql", "ingestEvents": "dataset:events:ingest", "submitIngestionSpec": "datasource:ingestion:submit", - "datasetExhaust": "dataset:exhaust:get" + "datasetExhaust": "dataset:exhaust:get", + "copyDataset": "dataset:copy", + "readConnectors": "connectors:read", + "listConnectors": "connectors:list", } \ No newline at end of file From ec87edc5e96bae83f6f518bc6db7ed7a26a508a7 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 9 Aug 2024 16:32:57 +0530 Subject: [PATCH 131/235] #OBS-I141: added telemetry for v2 api's --- api-service/src/routes/Router.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 444a318d..1faaafb0 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -48,8 +48,8 @@ router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); -router.post("/dataset/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/dataset/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); From 0b051375d32f3263f19eb99ceb02e195eb2c41d8 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Fri, 9 Aug 2024 17:08:37 +0530 Subject: [PATCH 132/235] #OBS-I143: feat: dataset publish changes to deploy flink connectors --- .../{flink => flink-connector}/Chart.lock | 0 .../{flink => flink-connector}/Chart.yaml | 0 .../charts/.helmignore | 0 .../charts/common/Chart.yaml | 0 .../charts/common/templates/_affinities.tpl | 0 .../charts/common/templates/_capabilities.tpl | 0 .../charts/common/templates/_configs.tpl | 0 .../charts/common/templates/_errors.tpl | 0 .../charts/common/templates/_images.tpl | 0 .../charts/common/templates/_ingress.tpl | 0 .../charts/common/templates/_labels.tpl | 0 .../charts/common/templates/_names.tpl | 0 .../charts/common/templates/_secrets.tpl | 0 .../charts/common/templates/_storage.tpl | 0 .../charts/common/templates/_tplvalues.tpl | 0 .../charts/common/templates/_utils.tpl | 0 .../charts/common/templates/_variables.tpl | 0 .../charts/common/templates/_warnings.tpl | 0 .../templates/validations/_cassandra.tpl | 0 .../common/templates/validations/_mariadb.tpl | 0 .../common/templates/validations/_mongodb.tpl | 0 .../common/templates/validations/_mysql.tpl | 0 .../templates/validations/_postgresql.tpl | 0 .../common/templates/validations/_redis.tpl | 0 .../templates/validations/_validations.tpl | 0 .../charts/common/values.yaml | 0 .../templates/NOTES.txt | 0 .../templates/_base_serviceAccount.tpl | 0 .../templates/_helpers.tpl | 0 .../templates/_image_flink.tpl | 7 +-- .../templates/_namespace.tpl | 0 .../templates/configmap.yaml | 0 .../templates/deployment.yaml | 44 +++++----------- .../templates/hpa.yaml | 0 .../templates/ingress.yaml | 0 .../templates/service.yaml | 0 .../templates/serviceaccount.yaml | 0 .../templates/servicemonitor.yaml | 0 .../{flink => flink-connector}/values.yaml | 10 +--- .../src/command/connector_command.py | 52 ++++++++++++++++++- command-service/src/config/service_config.yml | 2 +- 41 files changed, 69 insertions(+), 46 deletions(-) rename command-service/helm-charts/{flink => flink-connector}/Chart.lock (100%) rename command-service/helm-charts/{flink => flink-connector}/Chart.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/.helmignore (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/Chart.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_affinities.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_capabilities.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_configs.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_errors.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_images.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_ingress.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_labels.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_names.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_secrets.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_storage.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_tplvalues.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_utils.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_variables.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/_warnings.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_cassandra.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_mariadb.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_mongodb.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_mysql.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_postgresql.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_redis.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/templates/validations/_validations.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/charts/common/values.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/NOTES.txt (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/_base_serviceAccount.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/_helpers.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/_image_flink.tpl (57%) rename command-service/helm-charts/{flink => flink-connector}/templates/_namespace.tpl (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/configmap.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/deployment.yaml (83%) rename command-service/helm-charts/{flink => flink-connector}/templates/hpa.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/ingress.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/service.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/serviceaccount.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/templates/servicemonitor.yaml (100%) rename command-service/helm-charts/{flink => flink-connector}/values.yaml (97%) diff --git a/command-service/helm-charts/flink/Chart.lock b/command-service/helm-charts/flink-connector/Chart.lock similarity index 100% rename from command-service/helm-charts/flink/Chart.lock rename to command-service/helm-charts/flink-connector/Chart.lock diff --git a/command-service/helm-charts/flink/Chart.yaml b/command-service/helm-charts/flink-connector/Chart.yaml similarity index 100% rename from command-service/helm-charts/flink/Chart.yaml rename to command-service/helm-charts/flink-connector/Chart.yaml diff --git a/command-service/helm-charts/flink/charts/.helmignore b/command-service/helm-charts/flink-connector/charts/.helmignore similarity index 100% rename from command-service/helm-charts/flink/charts/.helmignore rename to command-service/helm-charts/flink-connector/charts/.helmignore diff --git a/command-service/helm-charts/flink/charts/common/Chart.yaml b/command-service/helm-charts/flink-connector/charts/common/Chart.yaml similarity index 100% rename from command-service/helm-charts/flink/charts/common/Chart.yaml rename to command-service/helm-charts/flink-connector/charts/common/Chart.yaml diff --git a/command-service/helm-charts/flink/charts/common/templates/_affinities.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_affinities.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_affinities.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_affinities.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_capabilities.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_capabilities.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_capabilities.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_capabilities.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_configs.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_configs.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_configs.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_configs.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_errors.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_errors.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_errors.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_errors.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_images.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_images.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_images.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_images.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_ingress.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_ingress.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_ingress.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_ingress.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_labels.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_labels.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_labels.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_labels.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_names.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_names.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_names.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_names.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_secrets.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_secrets.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_secrets.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_secrets.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_storage.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_storage.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_storage.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_storage.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_tplvalues.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_tplvalues.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_tplvalues.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_tplvalues.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_utils.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_utils.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_utils.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_utils.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_variables.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_variables.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_variables.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_variables.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/_warnings.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/_warnings.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/_warnings.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/_warnings.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_cassandra.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_cassandra.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_cassandra.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_cassandra.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_mariadb.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mariadb.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_mariadb.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_mariadb.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_mongodb.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mongodb.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_mongodb.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_mongodb.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_mysql.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_mysql.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_mysql.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_mysql.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_postgresql.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_postgresql.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_postgresql.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_postgresql.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_redis.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_redis.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_redis.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_redis.tpl diff --git a/command-service/helm-charts/flink/charts/common/templates/validations/_validations.tpl b/command-service/helm-charts/flink-connector/charts/common/templates/validations/_validations.tpl similarity index 100% rename from command-service/helm-charts/flink/charts/common/templates/validations/_validations.tpl rename to command-service/helm-charts/flink-connector/charts/common/templates/validations/_validations.tpl diff --git a/command-service/helm-charts/flink/charts/common/values.yaml b/command-service/helm-charts/flink-connector/charts/common/values.yaml similarity index 100% rename from command-service/helm-charts/flink/charts/common/values.yaml rename to command-service/helm-charts/flink-connector/charts/common/values.yaml diff --git a/command-service/helm-charts/flink/templates/NOTES.txt b/command-service/helm-charts/flink-connector/templates/NOTES.txt similarity index 100% rename from command-service/helm-charts/flink/templates/NOTES.txt rename to command-service/helm-charts/flink-connector/templates/NOTES.txt diff --git a/command-service/helm-charts/flink/templates/_base_serviceAccount.tpl b/command-service/helm-charts/flink-connector/templates/_base_serviceAccount.tpl similarity index 100% rename from command-service/helm-charts/flink/templates/_base_serviceAccount.tpl rename to command-service/helm-charts/flink-connector/templates/_base_serviceAccount.tpl diff --git a/command-service/helm-charts/flink/templates/_helpers.tpl b/command-service/helm-charts/flink-connector/templates/_helpers.tpl similarity index 100% rename from command-service/helm-charts/flink/templates/_helpers.tpl rename to command-service/helm-charts/flink-connector/templates/_helpers.tpl diff --git a/command-service/helm-charts/flink/templates/_image_flink.tpl b/command-service/helm-charts/flink-connector/templates/_image_flink.tpl similarity index 57% rename from command-service/helm-charts/flink/templates/_image_flink.tpl rename to command-service/helm-charts/flink-connector/templates/_image_flink.tpl index d0d7c2da..5c542f64 100644 --- a/command-service/helm-charts/flink/templates/_image_flink.tpl +++ b/command-service/helm-charts/flink-connector/templates/_image_flink.tpl @@ -3,12 +3,13 @@ {{- $context := .context }} {{- $scope := .scope }} {{- with $scope }} -{{- $registry := default $context.Values.global.image.registry .registry }} -{{- $image := printf "%s/%s" $registry .repository}} +{{- $registry := default $context.Values.registry .registry }} +{{- $repository := default $context.Values.repository .repository }} +{{- $image := printf "%s/%s" $registry $repository}} {{- if .digest }} {{- printf "%s@%s" $image .digest }} {{- else }} -{{- $tag := default "latest" .tag }} +{{- $tag := default $context.Values.tag .tag }} {{- printf "%s:%s" $image $tag }} {{- end }} {{- end }} diff --git a/command-service/helm-charts/flink/templates/_namespace.tpl b/command-service/helm-charts/flink-connector/templates/_namespace.tpl similarity index 100% rename from command-service/helm-charts/flink/templates/_namespace.tpl rename to command-service/helm-charts/flink-connector/templates/_namespace.tpl diff --git a/command-service/helm-charts/flink/templates/configmap.yaml b/command-service/helm-charts/flink-connector/templates/configmap.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/configmap.yaml rename to command-service/helm-charts/flink-connector/templates/configmap.yaml diff --git a/command-service/helm-charts/flink/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml similarity index 83% rename from command-service/helm-charts/flink/templates/deployment.yaml rename to command-service/helm-charts/flink-connector/templates/deployment.yaml index 1cd613b3..d6d7708f 100644 --- a/command-service/helm-charts/flink/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -51,37 +51,17 @@ spec: image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} workingDir: {{ .Values.taskmanager.flink_work_dir }} - # args: ["taskmanager"] - # env: - # - name: FLINK_PROPERTIES - # value: |+ - # jobmanager.rpc.address: {{ $jobName }}-jobmanager - # taskmanager.rpc.port=6122 - # taskmanager.numberOfTaskSlots: 2 - # metrics.reporters: prom - # metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory - # metrics.reporter.prom.host: {{ $jobName }}-taskmanager - # metrics.reporter.prom.port: 9251 - command: ["/opt/flink/bin/taskmanager.sh"] - args: ["start-foreground", - {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}", - {{- end }} - {{- if and (eq .Values.checkpoint_store_type "s3") (ne .Values.s3_auth_type "serviceAccount") }} - "-Ds3.access-key={{ .Values.s3_access_key }}", - "-Ds3.secret-key={{ .Values.s3_secret_key }}", - "-Ds3.endpoint={{ .Values.s3_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", - {{- end }} - {{- if eq .Values.checkpoint_store_type "gcs" }} - "-Dgoogle.cloud.auth.service.account.enable=true", - {{- end }} - "-Dweb.submit.enable=false", - "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", - "-Dmetrics.reporter.prom.host={{ $jobName }}-taskmanager", - "-Dmetrics.reporter.prom.port=9251-9260", - "-Djobmanager.rpc.address={{ $jobName }}-jobmanager", - "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"] + args: ["taskmanager"] + env: + - name: FLINK_PROPERTIES + value: |+ + jobmanager.rpc.address: {{ $jobName }}-jobmanager + taskmanager.rpc.port=6122 + taskmanager.numberOfTaskSlots: 2 + metrics.reporters: prom + metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory + metrics.reporter.prom.host: {{ $jobName }}-taskmanager + metrics.reporter.prom.port: 9251 ports: - containerPort: {{ .Values.taskmanager.rpc_port }} name: rpc @@ -140,7 +120,7 @@ spec: app.kubernetes.io/component: {{ printf "%s-%s" $jobName $component }} component: {{ printf "%s-%s" $jobName $component }} annotations: - checksum/config: {{ .Files.Glob "configs/*" | toYaml | sha256sum }} + checksum/config: {{ .Files.Glob "cFonfigs/*" | toYaml | sha256sum }} checksum/job-config: {{ $jobData | toYaml | sha256sum }} spec: serviceAccountName: {{ include "base.serviceaccountname" . }} diff --git a/command-service/helm-charts/flink/templates/hpa.yaml b/command-service/helm-charts/flink-connector/templates/hpa.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/hpa.yaml rename to command-service/helm-charts/flink-connector/templates/hpa.yaml diff --git a/command-service/helm-charts/flink/templates/ingress.yaml b/command-service/helm-charts/flink-connector/templates/ingress.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/ingress.yaml rename to command-service/helm-charts/flink-connector/templates/ingress.yaml diff --git a/command-service/helm-charts/flink/templates/service.yaml b/command-service/helm-charts/flink-connector/templates/service.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/service.yaml rename to command-service/helm-charts/flink-connector/templates/service.yaml diff --git a/command-service/helm-charts/flink/templates/serviceaccount.yaml b/command-service/helm-charts/flink-connector/templates/serviceaccount.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/serviceaccount.yaml rename to command-service/helm-charts/flink-connector/templates/serviceaccount.yaml diff --git a/command-service/helm-charts/flink/templates/servicemonitor.yaml b/command-service/helm-charts/flink-connector/templates/servicemonitor.yaml similarity index 100% rename from command-service/helm-charts/flink/templates/servicemonitor.yaml rename to command-service/helm-charts/flink-connector/templates/servicemonitor.yaml diff --git a/command-service/helm-charts/flink/values.yaml b/command-service/helm-charts/flink-connector/values.yaml similarity index 97% rename from command-service/helm-charts/flink/values.yaml rename to command-service/helm-charts/flink-connector/values.yaml index f0ea188c..84842bd9 100644 --- a/command-service/helm-charts/flink/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -321,15 +321,7 @@ serviceMonitor: jobLabel: "app.kubernetes.io/name" port: prom -flink_jobs: - kafka-connector: - enabled: true - registry: surabhi1510 - repository: flink-python-3.11 - tag: 1.0.0 - imagePullSecrets: [] - - job_classname: org.sunbird.obsrv.connector.KafkaConnector +flink_jobs: [] commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 2dbed28d..be82ae7b 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -86,12 +86,13 @@ def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata): def _install_jobs(self, dataset_id, active_connectors, is_masterdata): result = None - for connector in active_connectors: print("Installing connector {0}".format(connector)) if connector.connector_runtime == "spark": result = self._perform_spark_install(connector) + elif connector.connector_runtime == "flink": + result = self._perform_flink_install(connector) else: print( f"Connector {connector.connector_id} is not supported for deployment" @@ -104,6 +105,55 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): # for release in masterdata_jar_config: # result = self._perform_install(release) return result + + def _perform_flink_install(self, connector_instance): + err = None + result = None + release_name = connector_instance.id + connector_source = json.loads(connector_instance.connector_source) + flink_jobs = { + "kafka-connector": { + "enabled": "true", + "job_classname": connector_source.get('main_class') + } + } + set_json_value = json.dumps(flink_jobs) + print("Kafka connector: ", set_json_value) + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", + "--namespace", + self.connector_job_config["flink"]["namespace"], + "--create-namespace", + "--set-json", + f"flink_jobs={set_json_value}" + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} deployment succeeded...") + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", + ) + print( + f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + ) + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result def _perform_spark_install(self, connector_instance): err = None diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index a26d4a53..2c432ba0 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -172,7 +172,7 @@ connector_jobs: namespace: spark base_helm_chart: spark-connector-cron flink: - namespace: streaming-connectors + namespace: flink base_helm_chart: flink-connector connector_registry: From c09c6f3f70816be9095ce7b9f4dd6894af28790a Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 12 Aug 2024 15:02:24 +0530 Subject: [PATCH 133/235] #OBS-I141: removed metric for sum of response time --- api-service/src/metrics/prometheus/helpers.ts | 22 +++++-------------- api-service/src/metrics/prometheus/index.ts | 7 +++--- api-service/src/metrics/prometheus/metrics.ts | 10 +-------- 3 files changed, 9 insertions(+), 30 deletions(-) diff --git a/api-service/src/metrics/prometheus/helpers.ts b/api-service/src/metrics/prometheus/helpers.ts index 2f502d99..05051461 100644 --- a/api-service/src/metrics/prometheus/helpers.ts +++ b/api-service/src/metrics/prometheus/helpers.ts @@ -1,5 +1,5 @@ import { NextFunction, Response } from "express"; -import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime, incrementResponseTime } from "."; +import { incrementApiCalls, incrementFailedApiCalls, incrementSuccessfulApiCalls, setQueryResponseTime } from "."; import _ from "lodash"; import { Entity, Metric } from "../../types/MetricModel"; import { ObsrvError } from "../../types/ObsrvError"; @@ -20,10 +20,7 @@ export const onSuccess = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 200 } = res const labels = { ...metricLabels, status: statusCode } - if(duration){ - setQueryResponseTime({ duration, labels }) - incrementResponseTime({duration, labels}) - } + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementSuccessfulApiCalls({ labels }) } @@ -32,10 +29,7 @@ export const onFailure = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 500 } = res const labels = { ...metricLabels, status: statusCode } - if(duration){ - setQueryResponseTime({ duration, labels }) - incrementResponseTime({duration, labels}) - } + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } @@ -44,10 +38,7 @@ export const onGone = (req: any, res: Response) => { const { duration = 0, metricLabels }: Metric = getMetricLabels(req, res) const { statusCode = 410 } = res const labels = { ...metricLabels, status: statusCode } - if(duration){ - setQueryResponseTime({ duration, labels }) - incrementResponseTime({duration, labels}) - } + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } @@ -57,10 +48,7 @@ export const onObsrvFailure = (req: any, res: Response,error: ObsrvError) => { metricLabels.dataset_id = error.datasetId const { statusCode = 404 } = res const labels = { ...metricLabels, status: statusCode } - if(duration){ - setQueryResponseTime({ duration, labels }) - incrementResponseTime({duration, labels}) - } + duration && setQueryResponseTime({ duration, labels }) incrementApiCalls({ labels }) incrementFailedApiCalls({ labels }); } diff --git a/api-service/src/metrics/prometheus/index.ts b/api-service/src/metrics/prometheus/index.ts index 3fdefe19..173527ad 100644 --- a/api-service/src/metrics/prometheus/index.ts +++ b/api-service/src/metrics/prometheus/index.ts @@ -1,6 +1,6 @@ import client from "prom-client"; -import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric } from "./metrics" -const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric, sumResponseTimeMetric]; +import { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric } from "./metrics" +const metrics = [queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, successApiCallsMetric]; import { NextFunction } from "express"; const register = new client.Registry(); @@ -17,7 +17,6 @@ const incrementApiCalls = ({ labels = {} }: Record) => totalApiCall const setQueryResponseTime = ({ labels = {}, duration }: Record) => queryResponseTimeMetric.labels(labels).set(duration); const incrementFailedApiCalls = ({ labels = {} }: Record) => failedApiCallsMetric.labels(labels).inc(); const incrementSuccessfulApiCalls = ({ labels = {} }: Record) => successApiCallsMetric.labels(labels).inc(); -const incrementResponseTime = ({ labels = {}, duration }: Record) => sumResponseTimeMetric.labels(labels).inc(duration); //register the metrics configureRegistry(register); @@ -32,5 +31,5 @@ const metricsScrapeHandler = async (req: any, res: any, next: NextFunction) => { } } -export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls, incrementResponseTime}; +export { metricsScrapeHandler, incrementApiCalls, incrementFailedApiCalls, setQueryResponseTime, incrementSuccessfulApiCalls}; diff --git a/api-service/src/metrics/prometheus/metrics.ts b/api-service/src/metrics/prometheus/metrics.ts index 499bd15a..2985dc0f 100644 --- a/api-service/src/metrics/prometheus/metrics.ts +++ b/api-service/src/metrics/prometheus/metrics.ts @@ -28,17 +28,9 @@ const successApiCallsMetric = new Prometheus.Counter({ labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] }) -// Create a new Prometheus Counter for sum of response time -const sumResponseTimeMetric = new Prometheus.Counter({ - name: "node_sum_response_time", - help: "The sum of response time for time series of same label", - labelNames: ["entity", "id", "endpoint", "dataset_id", "status", "request_size", "response_size"] -}); - export { queryResponseTimeMetric, totalApiCallsMetric, failedApiCallsMetric, - successApiCallsMetric, - sumResponseTimeMetric + successApiCallsMetric } \ No newline at end of file From 59238bd2b8ee25a2783c3b7b86c1062da1ad54c4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 12 Aug 2024 15:05:13 +0530 Subject: [PATCH 134/235] #OBS-I141: removed usage of builtin kafka methods from telemetry file --- api-service/src/services/telemetry.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index e6139d94..d4eca61b 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -2,17 +2,13 @@ import { Request, Response, NextFunction } from "express" import { v4 } from "uuid"; import _ from "lodash"; import { config as appConfig } from "../configs/Config"; -import { Kafka } from "kafkajs"; +import {send} from "../connections/kafkaConnection" const {env, version} = _.pick(appConfig, ["env","version"]) const telemetryTopic = _.get(appConfig, "telemetry_dataset"); -const brokerServers = _.get(appConfig, "telemetry_service_config.kafka.config.brokers"); export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } -const kafka = new Kafka({ clientId: telemetryTopic, brokers: brokerServers }); -const telemetryEventsProducer = kafka.producer(); -telemetryEventsProducer.connect().catch(err => console.error("Unable to connect to kafka", err.message)); const getDefaults = () => { return { @@ -54,7 +50,7 @@ const getDefaultEdata = ({ action }: any) => ({ }) const sendTelemetryEvents = async (event: Record) => { - telemetryEventsProducer.send({ topic: telemetryTopic, messages: [{ value: JSON.stringify(event) }] }).catch(console.log) + send({ messages: [{ value: JSON.stringify(event) }] }, telemetryTopic).catch(console.log); } const transformProps = (body: Record) => { From 21a97a43e8a0ed2b8441d37244f99247730ef4a6 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 12 Aug 2024 17:34:53 +0530 Subject: [PATCH 135/235] #OBS-I146: feat: Retire fix --- .../DatasetRetire.spec.ts | 92 ++----------------- .../DatasetStatusTransition/Fixtures.ts | 75 +++++++-------- 2 files changed, 40 insertions(+), 127 deletions(-) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index c5c32929..0be1fdb9 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -29,7 +29,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { it("Dataset status transition success: When the action is to Retire dataset", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "dataset" }) + return Promise.resolve(TestInputsForDatasetStatusTransition.SCHEMA_TO_RETIRE) }) chai.spy.on(DatasetTransformations, "update", () => { return Promise.resolve({}) @@ -44,7 +44,7 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { return Promise.resolve({}) }) chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve(["telemetry"]) + return Promise.resolve([{ datasource_ref: "telemetry" }]) }) chai.spy.on(druidHttpService, "post", () => { return Promise.resolve({}) @@ -193,19 +193,19 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { it("Dataset status transition failure: When dataset is already retired", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Retired", type: "dataset" }) + return Promise.resolve({ ...TestInputsForDatasetStatusTransition.SCHEMA_TO_RETIRE, status: "Retired" }) }) chai .request(app) .post("/v2/datasets/status-transition") .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) .end((err, res) => { - res.should.have.status(httpStatus.BAD_REQUEST); + res.should.have.status(httpStatus.CONFLICT); res.body.should.be.a("object") res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to Retire dataset as it is not in live state") + res.body.error.message.should.be.eq("Transition failed for dataset: dataset-all-fields7 status:Retired with status transition to Retire") res.body.error.code.should.be.eq("DATASET_RETIRE_FAILURE") done(); }) @@ -213,19 +213,13 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { it("Dataset status transition failure: When dataset to retire is used by other datasets", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", type: "master-dataset", status: "Live" }) + return Promise.resolve({ ...TestInputsForDatasetStatusTransition.SCHEMA_TO_RETIRE, type: "master" }) }) chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "telemetry" }] } }]) + return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "dataset-all-fields7" }] } }]) }) chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "telemetry" }] } }]) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) + return Promise.resolve([{ dataset_id: "telemetry", denorm_config: { denorm_fields: [{ dataset_id: "dataset-all-fields7" }] } }]) }) chai .request(app) @@ -237,78 +231,10 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { res.body.id.should.be.eq("api.datasets.status-transition"); res.body.params.status.should.be.eq("FAILED") res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Failed to retire dataset as it is used by other datasets") + res.body.error.message.should.be.eq("Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset") res.body.error.code.should.be.eq("DATASET_IN_USE") done(); }); }); - it("Dataset status transition failure: When setting retire status to live records fail", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", status: "Live", type: "dataset" }) - }) - chai.spy.on(Dataset, "update", () => { - return Promise.reject({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq("api.datasets.status-transition"); - res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") - done(); - }); - }); - - it("Dataset status transition failure: Failed to restart pipeline", (done) => { - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ dataset_id: "telemetry", type: "dataset", status: "Live", }) - }) - chai.spy.on(DatasetTransformations, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(DatasetSourceConfig, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Dataset, "update", () => { - return Promise.resolve({}) - }) - chai.spy.on(Datasource, "findAll", () => { - return Promise.resolve(["telemetry"]) - }) - chai.spy.on(commandHttpService, "post", () => { - return Promise.reject({}) - }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_RETIRE) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq("api.datasets.status-transition"); - res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to perform status transition on datasets") - done(); - }); - }); }) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index db14c924..9a807e7f 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -86,51 +86,38 @@ export const TestInputsForDatasetStatusTransition = { "entry_topic": "local.ingest" }, INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { - "dataset_id": "telemetry", - "type": "", + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "event", "name": "sb-telemetry", - "id": "telemetry.1", - "status": "Draft", - "version_key": "1789887878", - "validation_config": { - "validate": true, - "mode": "Strict" - }, - "router_config": { - "topic": "test" - }, - "denorm_config": { - "redis_db_host": "local", - "redis_db_port": 5432, - "denorm_fields": [ - { - "denorm_key": "actor.id", - "denorm_out_field": "userdata", - "dataset_name": "name", - "dataset_id": "name" - }, - { - "denorm_key": "actor.id", - "denorm_out_field": "mid", - "dataset_name": "name", - "dataset_id": "name" - } - ] - }, - "dataset_config": { - "data_key": "mid", - "timestamp_key": "ets", - "entry_topic": "topic", - "redis_db_host": "local", - "redis_db_port": 5432, - "redis_db": 0, - "index_data": true - }, - "client_state": {}, - "tags": [ - "tag1", - "tag2" - ] + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "tags": ["tag1"], + "status":"Draft", + "version_key": "1721887933020", + "api_version": "v2" + }, + SCHEMA_TO_RETIRE: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "event", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "tags": ["tag1"], + "status":"Live", + "version_key": "1721887933020", + "api_version": "v2" }, DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } } \ No newline at end of file From 58643d41878fd04fb0adcd1b0fa998666f1a149e Mon Sep 17 00:00:00 2001 From: Aniket Sakinala Date: Tue, 13 Aug 2024 17:24:59 +0530 Subject: [PATCH 136/235] Issue #SBCOSS-12 fix: convert all SQL raw queries to prepared statements --- .../src/command/alert_manager_command.py | 10 +- .../src/command/connector_command.py | 31 +- .../src/command/connector_registry.py | 105 +++-- .../src/command/dataset_command.py | 17 +- command-service/src/command/db_command.py | 381 ++++++++++++------ command-service/src/command/druid_command.py | 3 +- command-service/src/service/db_service.py | 16 +- 7 files changed, 375 insertions(+), 188 deletions(-) diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py index 880fde1c..1c989a5c 100644 --- a/command-service/src/command/alert_manager_command.py +++ b/command-service/src/command/alert_manager_command.py @@ -81,13 +81,15 @@ def execute(self, command_payload: CommandPayload, action: Action): return ActionResponse(status="OK", status_code=200) def get_dataset(self, dataset_id: str) -> str: - query = f"SELECT * FROM datasets WHERE dataset_id='{dataset_id}'" - result = self.db_service.execute_select_one(sql=query) + query = f"SELECT * FROM datasets WHERE dataset_id= %s" + params = (dataset_id,) + result = self.db_service.execute_select_one(sql=query, params=params) return result def get_dataset_source_config(self, dataset_id: str) -> str: - query = f"SELECT * FROM dataset_source_config WHERE dataset_id='{dataset_id}'" - result = self.db_service.execute_select_all(sql=query) + query = f"SELECT * FROM dataset_source_config WHERE dataset_id= %s" + params = (dataset_id,) + result = self.db_service.execute_select_all(sql=query, params=params) return result def get_modified_metric( diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index be82ae7b..9b88a522 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -210,14 +210,14 @@ def _perform_spark_install(self, connector_instance): def _get_connector_details(self, dataset_id): active_connectors = [] - records = self.db_service.execute_select_all( - f""" - SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology - FROM connector_instances ci - JOIN connector_registry cr on ci.connector_id = cr.id - WHERE ci.status='{DatasetStatusType.Live.name}' and ci.dataset_id = '{dataset_id}' - """ - ) + query = f""" + SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology + FROM connector_instances ci + JOIN connector_registry cr on ci.connector_id = cr.id + WHERE ci.status= %s and ci.dataset_id = %s + """ + params = (DatasetStatusType.Live.name, dataset_id,) + records = self.db_service.execute_select_all(sql=query, params=params) for record in records: active_connectors.append(from_dict( @@ -228,14 +228,13 @@ def _get_connector_details(self, dataset_id): def _get_masterdata_details(self, dataset_id): is_masterdata = False - rows = self.db_service.execute_select_all( - f""" - SELECT * - FROM datasets - WHERE status='{DatasetStatusType.Live.name}' AND dataset_id = '{dataset_id}' AND type = 'master' - """ - ) - + query = f""" + SELECT * + FROM datasets + WHERE status= %s AND dataset_id = %s AND type = 'master' + """ + params = (DatasetStatusType.Live.name, dataset_id,) + rows = self.db_service.execute_select_all(sql=query, params=params) if len(rows) > 0: is_masterdata = True diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index 36cb0b57..ae185174 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -205,8 +205,8 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: 'SYSTEM', datetime.now().strftime("%Y-%m-%d %H:%M:%S") ) - query = self.build_insert_query(registry_meta) - success = self.execute_query(query) + query, params = self.build_insert_query(registry_meta) + success = self.execute_query(query, params) if not success: return RegistryResponse( status="failure", @@ -254,8 +254,8 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: 'SYSTEM', datetime.now().strftime("%Y-%m-%d %H:%M:%S") ) - query = self.build_insert_query(registry_meta) - success = self.execute_query(query) + query, params = self.build_insert_query(registry_meta) + success = self.execute_query(query, params) if not success: return RegistryResponse( status="failure", @@ -269,9 +269,9 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: statusCode=status.HTTP_200_OK, ) - def execute_query(self, query) -> bool: + def execute_query(self, query, params) -> bool: try: - result = self.db_service.execute_upsert(query) + result = self.db_service.execute_upsert(sql=query, params=params) return result > 0 # Assuming the result is the number of affected rows except Exception as e: print( @@ -307,28 +307,79 @@ def download_file(self, url, destination) -> bool: def build_insert_query(self, registry_meta: ConnectorRegsitryv2): ui_spec_json = json.dumps(registry_meta.ui_spec) - return f""" INSERT INTO connector_registry (id, connector_id, name, type, category, version, description, technology, runtime, licence, owner, iconurl, status, source_url, source, ui_spec, created_by, updated_by, created_date, updated_date, live_date) VALUES - ( '{registry_meta.id}-{registry_meta.version}', '{registry_meta.id}', '{registry_meta.name}', '{registry_meta.type}', '{registry_meta.category}', '{registry_meta.version}', '{registry_meta.description}', '{registry_meta.technology}', '{registry_meta.runtime}', '{registry_meta.licence}', '{registry_meta.owner}', '{registry_meta.iconurl}', '{registry_meta.status}', '{registry_meta.source_url}', '{registry_meta.source}', '{ui_spec_json}', 'SYSTEM', 'SYSTEM', '{datetime.now()}', '{datetime.now()}', '{datetime.now()}' ) - ON CONFLICT (connector_id, version) DO UPDATE - SET id = '{registry_meta.id}-{registry_meta.version}', - name = '{registry_meta.name}', - type = '{registry_meta.type}', - category = '{registry_meta.category}', - version = '{registry_meta.version}', - description = '{registry_meta.description}', - technology = '{registry_meta.technology}', - runtime = '{registry_meta.runtime}', - licence = '{registry_meta.licence}', - owner = '{registry_meta.owner}', - iconurl = '{registry_meta.iconurl}', - status = '{registry_meta.status}', - source_url = '{registry_meta.source_url}', - source = '{registry_meta.source}', - ui_spec = '{ui_spec_json}'::jsonb, - updated_date = '{datetime.now()}' - ;; + query =f""" + INSERT INTO connector_registry ( + id, connector_id, name, type, category, version, description, + technology, runtime, licence, owner, iconurl, status, source_url, + source, ui_spec, created_by, updated_by, created_date, updated_date, live_date + ) VALUES ( + %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s, %s, %s + ) ON CONFLICT ( + connector_id, version + ) DO UPDATE SET + id = %s, + name = %s, + type = %s, + category = %s, + version = %s, + description = %s, + technology = %s, + runtime = %s, + licence = %s, + owner = %s, + iconurl = %s, + status = %s, + source_url = %s, + source = %s, + ui_spec = %s::jsonb, + updated_date = %s + ;; """ - + params = ( + registry_meta.id + "-" + registry_meta.version, + registry_meta.id, + registry_meta.name, + registry_meta.type, + registry_meta.category, + registry_meta.version, + registry_meta.description, + + registry_meta.technology, + registry_meta.runtime, + registry_meta.licence, + registry_meta.owner, + registry_meta.iconurl, + registry_meta.status, + registry_meta.source_url, + + registry_meta.source, + ui_spec_json, + 'SYSTEM', + 'SYSTEM', + datetime.now(), + datetime.now(), + datetime.now(), + + registry_meta.id, + registry_meta.name, + registry_meta.type, + registry_meta.category, + registry_meta.version, + registry_meta.description, + registry_meta.technology, + registry_meta.runtime, + registry_meta.licence, + registry_meta.owner, + registry_meta.iconurl, + registry_meta.status, + registry_meta.source_url, + registry_meta.source, + ui_spec_json, + datetime.now(), + ) + return query, params class ExtractionUtil: def extract_gz(tar_path, extract_path): diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index 165efd38..db7afa7d 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -30,9 +30,9 @@ def __init__( def _get_draft_dataset_record(self, dataset_id): query = f""" - SELECT "type", MAX(version) AS max_version FROM datasets_draft WHERE dataset_id = '{dataset_id}' GROUP BY 1 + SELECT "type", MAX(version) AS max_version FROM datasets_draft WHERE dataset_id = %s GROUP BY 1 """ - dataset_record = self.db_service.execute_select_one(query) + dataset_record = self.db_service.execute_select_one(sql=query, params=(dataset_id,)) if dataset_record is not None: return dataset_record return None @@ -40,19 +40,22 @@ def _get_draft_dataset_record(self, dataset_id): def _get_draft_dataset(self, dataset_id): query = f""" SELECT * FROM datasets_draft - WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}') AND version = (SELECT MAX(version) - FROM datasets_draft WHERE dataset_id = '{dataset_id}' AND (status = '{DatasetStatusType.Publish.name}' OR status = '{DatasetStatusType.ReadyToPublish.name}')) + WHERE dataset_id = %s AND (status = %s OR status = %s ) AND version = (SELECT MAX(version) + FROM datasets_draft WHERE dataset_id = %s AND (status = %s OR status = %s )) """ - dataset_record = self.db_service.execute_select_one(query) + params = (dataset_id, DatasetStatusType.Publish.name, DatasetStatusType.ReadyToPublish.name, + dataset_id, DatasetStatusType.Publish.name, DatasetStatusType.ReadyToPublish.name,) + dataset_record = self.db_service.execute_select_one(sql=query, params=params) if dataset_record is not None: return dataset_record return None def _check_for_live_record(self, dataset_id): query = f""" - SELECT * FROM datasets WHERE dataset_id = '{dataset_id}' AND status = '{DatasetStatusType.Live.name}' + SELECT * FROM datasets WHERE dataset_id = %s AND status = %s """ - result = self.db_service.execute_select_one(query) + params = (dataset_id, DatasetStatusType.Live.name, ) + result = self.db_service.execute_select_one(sql=query, params=params) live_dataset = dict() if result is not None: live_dataset = from_dict(data_class=DatasetsLive, data=result) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 6d2c7d46..831300be 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -63,56 +63,99 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d draft_dataset = from_dict(data_class = DatasetsDraft, data = draft_dataset_record) draft_dataset_id = draft_dataset.id current_timestamp = dt.now() + params = ( + dataset_id, + dataset_id, + draft_dataset.type, + draft_dataset.name, + json.dumps(draft_dataset.extraction_config).replace("'", "''"), + json.dumps(draft_dataset.validation_config).replace("'", "''"), + json.dumps(draft_dataset.dedup_config).replace("'", "''"), + json.dumps(draft_dataset.denorm_config).replace("'", "''"), + json.dumps(draft_dataset.data_schema).replace("'", "''"), + json.dumps(draft_dataset.router_config).replace("'", "''"), + json.dumps(draft_dataset.dataset_config).replace("'", "''"), + DatasetStatusType.Live.name, + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}"), + draft_dataset.api_version, + draft_dataset.version, + json.dumps(draft_dataset.sample_data).replace("'", "''"), + draft_dataset.entry_topic, + draft_dataset.created_by, + draft_dataset.updated_by, + current_timestamp, + current_timestamp, + current_timestamp, + + draft_dataset.name, + json.dumps(draft_dataset.extraction_config).replace("'", "''"), + json.dumps(draft_dataset.validation_config).replace("'", "''"), + json.dumps(draft_dataset.dedup_config).replace("'", "''"), + json.dumps(draft_dataset.denorm_config).replace("'", "''"), + json.dumps(draft_dataset.data_schema).replace("'", "''"), + json.dumps(draft_dataset.router_config).replace("'", "''"), + json.dumps(draft_dataset.dataset_config).replace("'", "''"), + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}"), + data_version if live_dataset is not None else 1, + draft_dataset.api_version, + draft_dataset.version, + json.dumps(draft_dataset.sample_data).replace("'", "''"), + draft_dataset.entry_topic, + draft_dataset.updated_by, + current_timestamp, + current_timestamp, + DatasetStatusType.Live.name, + ) insert_query = f""" INSERT INTO datasets(id, dataset_id, "type", name, extraction_config, validation_config, dedup_config, denorm_config, data_schema, router_config, dataset_config, status, tags, data_version, api_version, version, sample_data, entry_topic, created_by, updated_by, created_date, updated_date, published_date) VALUES ( - '{dataset_id}', - '{dataset_id}', - '{draft_dataset.type}', - '{draft_dataset.name}', - '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', - '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', - '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', - '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', - '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', - '{json.dumps(draft_dataset.router_config).replace("'", "''")}', - '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', - '{DatasetStatusType.Live.name}', - '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, 1, - '{draft_dataset.api_version}', - {draft_dataset.version}, - '{json.dumps(draft_dataset.sample_data).replace("'", "''")}', - '{draft_dataset.entry_topic}', - '{draft_dataset.created_by}', - '{draft_dataset.updated_by}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}' + %s, + %d, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) ON CONFLICT (id) DO UPDATE - SET name = '{draft_dataset.name}', - extraction_config = '{json.dumps(draft_dataset.extraction_config).replace("'", "''")}', - validation_config = '{json.dumps(draft_dataset.validation_config).replace("'", "''")}', - dedup_config = '{json.dumps(draft_dataset.dedup_config).replace("'", "''")}', - denorm_config = '{json.dumps(draft_dataset.denorm_config).replace("'", "''")}', - data_schema = '{json.dumps(draft_dataset.data_schema).replace("'", "''")}', - router_config = '{json.dumps(draft_dataset.router_config).replace("'", "''")}', - dataset_config = '{json.dumps(draft_dataset.dataset_config).replace("'", "''")}', - tags = '{json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}")}', - data_version = {data_version if live_dataset is not None else 1}, - api_version = '{draft_dataset.api_version}', - version = {draft_dataset.version}, - sample_data = '{json.dumps(draft_dataset.sample_data).replace("'", "''")}', - entry_topic = '{draft_dataset.entry_topic}', - updated_by = '{draft_dataset.updated_by}', - updated_date = '{current_timestamp}', - published_date = '{current_timestamp}', - status = '{DatasetStatusType.Live.name}'; + SET name = %s, + extraction_config = %s, + validation_config = %s, + dedup_config = %s, + denorm_config = %s, + data_schema = %s, + router_config = %s, + dataset_config = %s, + tags = %s, + data_version = %d, + api_version = %s, + version = %d, + sample_data = %s, + entry_topic = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + status = %s; """ - self.db_service.execute_upsert(insert_query) + self.db_service.execute_upsert(insert_query, params) print(f"Dataset {dataset_id} record inserted successfully...") return draft_dataset_id @@ -120,51 +163,84 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): result = {} draft_datasource_record = self.db_service.execute_select_all( - f"SELECT * FROM datasources_draft WHERE dataset_id = '{draft_dataset_id}'" + sql=f"SELECT * FROM datasources_draft WHERE dataset_id = %s", + params=(draft_dataset_id,) ) if draft_datasource_record is None: return result for record in draft_datasource_record: draft_datasource = from_dict(data_class=DatasourcesDraft, data=record) current_timestamp = dt.now() + params = ( + draft_datasource.id, + draft_datasource.datasource, + dataset_id, + draft_datasource.datasource_ref, + json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + draft_datasource.type, + json.dumps(draft_datasource.retention_period).replace("'", "''"), + json.dumps(draft_datasource.archival_policy).replace("'", "''"), + json.dumps(draft_datasource.purge_policy).replace("'", "''"), + json.dumps(draft_datasource.backup_config).replace("'", "''"), + DatasetStatusType.Live.name, + draft_datasource.created_by, + draft_datasource.updated_by, + current_timestamp, + current_timestamp, + current_timestamp, + json.dumps(draft_datasource.metadata).replace("'", "''"), + + draft_datasource.datasource, + json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + draft_datasource.type, + json.dumps(draft_datasource.retention_period).replace("'", "''"), + json.dumps(draft_datasource.archival_policy).replace("'", "''"), + json.dumps(draft_datasource.purge_policy).replace("'", "''"), + json.dumps(draft_datasource.backup_config).replace("'", "''"), + draft_datasource.updated_by, + current_timestamp, + current_timestamp, + json.dumps(draft_datasource.metadata).replace("'", "''"), + DatasetStatusType.Live.name, + ) insert_query = f""" INSERT INTO datasources(id, datasource, dataset_id, datasource_ref, ingestion_spec, type, retention_period, archival_policy, purge_policy, backup_config, status, created_by, updated_by, created_date, updated_date, published_date, metadata) VALUES ( - '{draft_datasource.id}', - '{draft_datasource.datasource}', - '{dataset_id}', - '{draft_datasource.datasource_ref}', - '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', - '{draft_datasource.type}', - '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', - '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', - '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', - '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', - '{DatasetStatusType.Live.name}', - '{draft_datasource.created_by}', - '{draft_datasource.updated_by}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}', - '{json.dumps(draft_datasource.metadata).replace("'", "''")}' + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) ON CONFLICT (id) DO UPDATE - SET datasource_ref = '{draft_datasource.datasource}', - ingestion_spec = '{json.dumps(draft_datasource.ingestion_spec).replace("'", "''")}', - type = '{draft_datasource.type}', - retention_period = '{json.dumps(draft_datasource.retention_period).replace("'", "''")}', - archival_policy = '{json.dumps(draft_datasource.archival_policy).replace("'", "''")}', - purge_policy = '{json.dumps(draft_datasource.purge_policy).replace("'", "''")}', - backup_config = '{json.dumps(draft_datasource.backup_config).replace("'", "''")}', - updated_by = '{draft_datasource.updated_by}', - updated_date = '{current_timestamp}', - published_date = '{current_timestamp}', - metadata = '{json.dumps(draft_datasource.metadata).replace("'", "''")}', - status = '{DatasetStatusType.Live.name}'; + SET datasource_ref = %s, + ingestion_spec = %s, + type = %s, + retention_period = %s, + archival_policy = %s, + purge_policy = %s, + backup_config = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + metadata = %s, + status = %s; """ - result = self.db_service.execute_upsert(insert_query) + result = self.db_service.execute_upsert(sql=insert_query, params=params) print( f"Datasource {draft_datasource.id} record inserted successfully..." ) @@ -183,63 +259,105 @@ def _insert_connector_instances(self, dataset_id, draft_dataset_record): ) current_timestamp = dt.now() if connector_config.version == 'v2': + params = ( + connector_config.id, + dataset_id, + connector_config.connector_id, + json.dumps(connector_config.connector_config).replace("'", "''"), + json.dumps(connector_config.operations_config).replace("'", "''"), + connector_config.data_format, + DatasetStatusType.Live.name, + json.dumps(emptyJson), + json.dumps(emptyJson), + draft_dataset_record.get('created_by'), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + current_timestamp, + + json.dumps(connector_config.connector_config).replace("'", "''"), + json.dumps(connector_config.operations_config).replace("'", "''"), + connector_config.data_format, + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + DatasetStatusType.Live.name, + ) insert_query = f""" INSERT INTO connector_instances(id, dataset_id, connector_id, connector_config, operations_config, data_format, status, connector_state, connector_stats, created_by, updated_by, created_date, updated_date, published_date) VALUES ( - '{connector_config.id}', - '{dataset_id}', - '{connector_config.connector_id}', - '{json.dumps(connector_config.connector_config).replace("'", "''")}', - '{json.dumps(connector_config.operations_config).replace("'", "''")}', - '{connector_config.data_format}', - '{DatasetStatusType.Live.name}', - '{json.dumps(emptyJson)}', - '{json.dumps(emptyJson)}', - '{draft_dataset_record.get('created_by')}', - '{draft_dataset_record.get('updated_by')}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}' + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) ON CONFLICT (id) DO UPDATE - SET connector_config = '{json.dumps(connector_config.connector_config).replace("'", "''")}', - operations_config = '{json.dumps(connector_config.operations_config).replace("'", "''")}', - data_format = '{connector_config.data_format}', - updated_by = '{draft_dataset_record.get('updated_by')}', - updated_date = '{current_timestamp}', - published_date = '{current_timestamp}', - status = '{DatasetStatusType.Live.name}'; + SET connector_config = %s, + operations_config = %s, + data_format = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + status = %s; """ - result = self.db_service.execute_upsert(insert_query) + result = self.db_service.execute_upsert(sql=insert_query, params=params) print( f"Connector[v2] Instance record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." ) else: + params = ( + connector_config.id, + dataset_id, + connector_config.connector_id, + json.dumps(connector_config.connector_config).replace("'", "''"), + DatasetStatusType.Live.name, + draft_dataset_record.get('created_by'), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + current_timestamp, + + json.dumps(connector_config.connector_config).replace("'", "''"), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + DatasetStatusType.Live.name, + ) insert_query = f""" INSERT INTO dataset_source_config(id, dataset_id, connector_type, connector_config, status, created_by, updated_by, created_date, updated_date, published_date) VALUES ( - '{connector_config.id}', - '{dataset_id}', - '{connector_config.connector_id}', - '{json.dumps(connector_config.connector_config).replace("'", "''")}', - '{DatasetStatusType.Live.name}', - '{draft_dataset_record.get('created_by')}', - '{draft_dataset_record.get('updated_by')}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}' + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) ON CONFLICT (id) DO UPDATE - SET connector_config = '{json.dumps(connector_config.connector_config).replace("'", "''")}', - updated_by = '{draft_dataset_record.get('updated_by')}', - updated_date = '{current_timestamp}', - published_date = '{current_timestamp}', - status = '{DatasetStatusType.Live.name}'; + SET connector_config = %s, + updated_by = %s, + updated_date = %s, + published_date = %s, + status = %s; """ - result = self.db_service.execute_upsert(insert_query) + result = self.db_service.execute_upsert(sql=insert_query, params=params) print( f"Connector[v1] record for [dataset={dataset_id},connector={connector_config.connector_id},id={connector_config.id}] inserted successfully..." ) @@ -252,7 +370,7 @@ def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): result = {} current_timestamp = dt.now() # Delete existing transformations - self.db_service.execute_delete(f"""DELETE from dataset_transformations where dataset_id = '{dataset_id}'""") + self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations where dataset_id = %s""", params=(dataset_id,)) print(f"Dataset Transformation for {dataset_id} are deleted successfully...") if draft_dataset_transformations_record is None: @@ -262,31 +380,44 @@ def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): transformation = from_dict( data_class=DatasetTransformationsDraft, data=record ) + params = ( + dataset_id + '_' + transformation.field_key, + dataset_id, + transformation.field_key, + json.dumps(transformation.transformation_function).replace("'", "''"), + DatasetStatusType.Live.name, + transformation.mode, + draft_dataset_record.get('created_by'), + draft_dataset_record.get('updated_by'), + current_timestamp, + current_timestamp, + current_timestamp, + ) insert_query = f""" INSERT INTO dataset_transformations(id, dataset_id, field_key, transformation_function, status, mode, created_by, updated_by, created_date, updated_date, published_date) VALUES ( - '{dataset_id + '_' + transformation.field_key}', - '{dataset_id}', - '{transformation.field_key}', - '{json.dumps(transformation.transformation_function).replace("'", "''")}', - '{DatasetStatusType.Live.name}', - '{transformation.mode}', - '{draft_dataset_record.get('created_by')}', - '{draft_dataset_record.get('updated_by')}', - '{current_timestamp}', - '{current_timestamp}', - '{current_timestamp}' + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s, + %s ) """ - result = self.db_service.execute_upsert(insert_query) + result = self.db_service.execute_upsert(sql=insert_query, params=params) print(f"Dataset Transformation {dataset_id + '_' + transformation.field_key} record inserted successfully...") return result def _delete_draft_dataset(self, dataset_id, draft_dataset_id): - self.db_service.execute_delete(f"""DELETE from datasources_draft where dataset_id = '{draft_dataset_id}'""") + self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") - self.db_service.execute_delete(f"""DELETE from datasets_draft where id = '{draft_dataset_id}'""") + self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) print(f"Draft Dataset for {dataset_id} is deleted successfully...") \ No newline at end of file diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py index 1bd95ab2..901b18ad 100644 --- a/command-service/src/command/druid_command.py +++ b/command-service/src/command/druid_command.py @@ -27,7 +27,8 @@ def execute(self, command_payload: CommandPayload, action: Action): def _submit_ingestion_task(self, dataset_id): datasources_records = self.db_service.execute_select_all( - f"SELECT dso.*, dt.type as dataset_type FROM datasources dso, datasets dt WHERE dso.dataset_id = '{dataset_id}' AND dso.dataset_id = dt.id" + sql=f"SELECT dso.*, dt.type as dataset_type FROM datasources dso, datasets dt WHERE dso.dataset_id = %s AND dso.dataset_id = dt.id", + params=(dataset_id,) ) if datasources_records is not None: print( diff --git a/command-service/src/service/db_service.py b/command-service/src/service/db_service.py index 864b4612..ded56b91 100644 --- a/command-service/src/service/db_service.py +++ b/command-service/src/service/db_service.py @@ -38,36 +38,36 @@ def connect(self): return db_connection # @reconnect - def execute_select_one(self, sql): + def execute_select_one(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute(sql) + cursor.execute(sql, params) result = cursor.fetchone() db_connection.close() return result # @reconnect - def execute_select_all(self, sql): + def execute_select_all(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute(sql) + cursor.execute(sql, params) result = cursor.fetchall() db_connection.close() return result # @reconnect - def execute_upsert(self, sql): + def execute_upsert(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute(sql) + cursor.execute(sql, params) record_count = cursor.rowcount db_connection.close() # print(f"{record_count} inserted/updated successfully") return record_count # @reconnect - def execute_delete(self, sql): + def execute_delete(self, sql, params): db_connection = self.connect() cursor = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute(sql) + cursor.execute(sql, params) db_connection.close() From 2f2f7557d26137e73850085ff4ee85b1ed9b38e7 Mon Sep 17 00:00:00 2001 From: Aniket Sakinala Date: Mon, 19 Aug 2024 18:23:42 +0530 Subject: [PATCH 137/235] Issue #SBCOSS-12 fix: tags is an array, so requires empty json for null case; dataset draft deletion requires deletion of transformation and source config drafts --- command-service/src/command/db_command.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 831300be..4cb62e2c 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -76,7 +76,7 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d json.dumps(draft_dataset.router_config).replace("'", "''"), json.dumps(draft_dataset.dataset_config).replace("'", "''"), DatasetStatusType.Live.name, - json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}"), + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}") if draft_dataset.tags is not None else json.dumps({}), draft_dataset.api_version, draft_dataset.version, json.dumps(draft_dataset.sample_data).replace("'", "''"), @@ -95,7 +95,7 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d json.dumps(draft_dataset.data_schema).replace("'", "''"), json.dumps(draft_dataset.router_config).replace("'", "''"), json.dumps(draft_dataset.dataset_config).replace("'", "''"), - json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}"), + json.dumps(draft_dataset.tags).replace("'", "''").replace("[", "{").replace("]", "}") if draft_dataset.tags is not None else json.dumps({}), data_version if live_dataset is not None else 1, draft_dataset.api_version, draft_dataset.version, @@ -126,7 +126,7 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d %s, 1, %s, - %d, + %s, %s, %s, %s, @@ -145,9 +145,9 @@ def _insert_dataset_record(self, dataset_id, data_version, live_dataset, draft_d router_config = %s, dataset_config = %s, tags = %s, - data_version = %d, + data_version = %s, api_version = %s, - version = %d, + version = %s, sample_data = %s, entry_topic = %s, updated_by = %s, @@ -419,5 +419,11 @@ def _delete_draft_dataset(self, dataset_id, draft_dataset_id): self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") + self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft transformations/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_source_config_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft source config/tables for {dataset_id} are deleted successfully...") + self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) print(f"Draft Dataset for {dataset_id} is deleted successfully...") \ No newline at end of file From 727dd2f92422bfa2a549e8b8159199453cab78e3 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Tue, 20 Aug 2024 11:30:25 +0530 Subject: [PATCH 138/235] #SBCOSS-23: feat: dataset publish changes for redeployment --- .../flink-connector/templates/deployment.yaml | 44 ++- .../helm-charts/flink-connector/values.yaml | 23 +- .../src/command/connector_command.py | 340 +++++++++++++----- 3 files changed, 280 insertions(+), 127 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index d6d7708f..946ca29d 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -69,9 +69,12 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - - name: flink-config-volume - mountPath: /data/flink/conf/flink-connector.conf - subPath: flink-connector.conf + # - name: flink-config-volume + # mountPath: /data/flink/conf/connectors-scala-config.conf + # subPath: connectors-scala-config.conf + # - name: flink-config-volume + # mountPath: /data/flink/conf/connectors-python-config.conf + # subPath: connectors-python-config.yaml # - name: flink-config-volume # mountPath: /opt/flink/conf/flink-conf.yaml # subPath: flink-conf.yaml @@ -79,12 +82,14 @@ spec: mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties volumes: - - name: flink-config-volume - configMap: - name: flink-connector-conf - items: - - key: connectors-scala-config.conf - path: flink-connector.conf + # - name: flink-config-volume + # configMap: + # name: flink-connector-conf + # items: + # - key: connectors-scala-config.conf + # path: connectors-scala-config.conf + # - key: connectors-python-config.yaml + # path: connectors-python-config.yaml - name: flink-common-volume configMap: name: {{ $jobName }}-config @@ -151,17 +156,19 @@ spec: metrics.reporter.prom.factory.class: org.apache.flink.metrics.prometheus.PrometheusReporterFactory metrics.reporter.prom.host: {{ $jobName }}-jobmanager metrics.reporter.prom.port: 9250 - volumeMounts: - name: flink-config-volume - mountPath: /data/flink/conf/flink-connector.conf - subPath: flink-connector.conf + mountPath: /data/flink/conf/connectors-scala-config.conf + subPath: connectors-scala-config.conf + - name: flink-config-volume + mountPath: /data/flink/conf/connectors-python-config.conf + subPath: connectors-python-config.conf - name: data mountPath: /flink/connectors - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties - + - name: {{ $jobName }}-job-submit image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} @@ -199,8 +206,11 @@ spec: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - name: flink-config-volume - mountPath: /data/flink/conf/flink-connector.conf - subPath: flink-connector.conf + mountPath: /data/flink/conf/connectors-scala-config.conf + subPath: connectors-scala-config.conf + - name: flink-config-volume + mountPath: /data/flink/conf/connectors-python-config.yaml + subPath: connectors-python-config.yaml - name: data mountPath: /flink/connectors - name: flink-common-volume @@ -214,7 +224,9 @@ spec: name: flink-connector-conf items: - key: connectors-scala-config.conf - path: flink-connector.conf + path: connectors-scala-config.conf + - key: connectors-python-config.yaml + path: connectors-python-config.yaml - name: flink-common-volume configMap: name: {{ $jobName }}-config diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index 84842bd9..b1c48924 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -264,27 +264,6 @@ baseconfig: | port = "{{ .Values.global.cassandra.port }}" } -# !!! Don't override the resources here. It's just a template -# Proper way to override resouce is to -# flink_jobs: -# master-data-processor: -# resources: -# taskmanager: -# resources: -# requests: -# cpu: 100m -# memory: 100Mi -# limits: -# cpu: 1 -# memory: 1024Mi -# jobmanager: -# resources: -# requests: -# cpu: 100m -# memory: 100Mi -# limits: -# cpu: 1 -# memory: 1024Mi flink_resources: taskmanager: resources: @@ -303,7 +282,7 @@ flink_resources: cpu: 1 memory: 1024Mi -serviceMonitor: +serviceMonitor: jobmanager: enabled: true interval: 30s diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index be82ae7b..c24d6408 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -38,12 +38,16 @@ def execute(self, command_payload: CommandPayload, action: Action): def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): result = None - # self._stop_connector_jobs(dataset_id, active_connectors, is_masterdata) + namespaces = set() + for job_type, config in self.connector_job_config.items(): + namespaces.add(config["namespace"]) + for namespace in namespaces: + self._stop_connector_jobs(dataset_id, active_connectors, is_masterdata, namespace) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) return result - def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata): + def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata, namespace): managed_releases = [] connector_jar_config = self.config.find("connector_job") masterdata_jar_config = self.config.find("masterdata_job") @@ -54,7 +58,7 @@ def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata): for release in masterdata_jar_config: managed_releases.append(release["release_name"]) - helm_ls_cmd = ["helm", "ls", "--namespace", self.connector_job_ns] + helm_ls_cmd = ["helm", "ls", "--namespace", namespace] helm_ls_result = subprocess.run( helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) @@ -90,15 +94,15 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): print("Installing connector {0}".format(connector)) if connector.connector_runtime == "spark": - result = self._perform_spark_install(connector) + result = self._perform_spark_install(dataset_id, connector) elif connector.connector_runtime == "flink": - result = self._perform_flink_install(connector) + result = self._perform_flink_install(dataset_id, connector) else: print( f"Connector {connector.connector_id} is not supported for deployment" ) break - + # if is_masterdata: # print("Installing masterdata job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -106,113 +110,235 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): # result = self._perform_install(release) return result - def _perform_flink_install(self, connector_instance): + def _perform_flink_install(self, dataset_id, connector_instance): err = None result = None - release_name = connector_instance.id - connector_source = json.loads(connector_instance.connector_source) - flink_jobs = { - "kafka-connector": { - "enabled": "true", - "job_classname": connector_source.get('main_class') - } - } - set_json_value = json.dumps(flink_jobs) - print("Kafka connector: ", set_json_value) - helm_install_cmd = [ - "helm", - "upgrade", - "--install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", - "--namespace", - self.connector_job_config["flink"]["namespace"], - "--create-namespace", - "--set-json", - f"flink_jobs={set_json_value}" - ] + release_name = connector_instance.connector_id + runtime = connector_instance.connector_runtime + namespace = self.connector_job_config["flink"]["namespace"] - print(" ".join(helm_install_cmd)) - - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + helm_ls_cmd = ["helm", "ls", "--namespace", namespace] + + helm_ls_result = subprocess.run( + helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} deployment succeeded...") + + if helm_ls_result.returncode == 0: + jobs = helm_ls_result.stdout.decode() + print(jobs) + deployment_exists = any(release_name in line for line in jobs.splitlines()[1:]) + if deployment_exists: + restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={release_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={release_name}-taskmanager --namespace {namespace}".format( + namespace=namespace, release_name=release_name + ) + print("Restart command: ", restart_cmd) + # Run the helm command + helm_install_result = subprocess.run( + restart_cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=True, + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} re-deployment succeeded...") + else: + err = True + return ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_HELM_LIST_EXCEPTION", + ) + print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result + else: + connector_source = json.loads(connector_instance.connector_source) + flink_jobs = { + "kafka-connector": { + "enabled": "true", + "job_classname": connector_source.get('main_class') + } + } + set_json_value = json.dumps(flink_jobs) + print("Kafka connector: ", set_json_value) + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set-json", + f"flink_jobs={set_json_value}" + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} deployment succeeded...") + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", + ) + print( + f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + ) + + if err is None: + result = ActionResponse(status="OK", status_code=200) + + return result else: - err = True - result = ActionResponse( + print(f"Error checking Flink deployments: {helm_ls_result.stderr.decode()}") + return ActionResponse( status="ERROR", status_code=500, - error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", + error_message="FLINK_HELM_LIST_EXCEPTION", ) - print( - f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" - ) - - if err is None: - result = ActionResponse(status="OK", status_code=200) - - return result - - def _perform_spark_install(self, connector_instance): + + def _perform_spark_install(self, dataset_id, connector_instance): err = None result = None release_name = connector_instance.id - # print("Instance -->>", connector_instance) connector_source = json.loads(connector_instance.connector_source) - print(connector_source) - helm_install_cmd = [ - "helm", - "upgrade", - "--install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", - "--namespace", - self.connector_job_config["spark"]["namespace"], - "--create-namespace", - "--set", - "technology={}".format(connector_instance.technology), - "--set", - "instance_id={}".format(release_name), - "--set", - "connector_source={}".format(connector_source["source"]), - "--set", - "main_class={}".format(connector_source["main_class"]), - "--set", - "main_file={}".format(connector_source["main_program"]), - "--set", - "cronSchedule={}".format(connector_instance.operations_config["schedule"]) - ] - - print(" ".join(helm_install_cmd)) - - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} deployment succeeded...") + schedule = connector_instance.operations_config["schedule"] + + schedule_configs = { + "Hourly": "0 * * * *", # Runs at the start of every hour + "Weekly": "0 0 * * 0", # Runs at midnight every Sunday + "Monthly": "0 0 1 * *", # Runs at midnight on the 1st day of every month + "Yearly": "0 0 1 1 *" # Runs at midnight on January 1st each year + } + + # Define namespace + namespace = self.connector_job_config["spark"]["namespace"] + + # Check if the job already exists + helm_ls_cmd = ["helm", "ls", "--namespace", namespace] + helm_ls_result = subprocess.run(helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + if helm_ls_result.returncode == 0: + jobs = helm_ls_result.stdout.decode() + job_exists = any(release_name in line for line in jobs.splitlines()[1:]) + + if job_exists: + if self._is_dataset_retired(dataset_id): + print("Dataset is retired. Uninstalling existing cron job.") + self._stop_connector_jobs(dataset_id, [], False, namespace) + else: + print("Updating existing job") + + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set", + "technology={}".format(connector_instance.technology), + "--set", + "instance_id={}".format(release_name), + "--set", + "connector_source={}".format(connector_source["source"]), + "--set", + "main_class={}".format(connector_source["main_class"]), + "--set", + "main_file={}".format(connector_source["main_program"]), + "--set", + "cronSchedule={}".format(schedule_configs[schedule]) + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} update succeeded...") + result = ActionResponse(status="OK", status_code=200) + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", + ) + print(f"Error updating job {release_name}: {helm_install_result.stderr.decode()}") + else: + print("Installing new job") + + helm_install_cmd = [ + "helm", + "install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set", + "technology={}".format(connector_instance.technology), + "--set", + "instance_id={}".format(release_name), + "--set", + "connector_source={}".format(connector_source["source"]), + "--set", + "main_class={}".format(connector_source["main_class"]), + "--set", + "main_file={}".format(connector_source["main_program"]), + "--set", + "cronSchedule={}".format(schedule_configs[schedule]) + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} installation succeeded...") + result = ActionResponse(status="OK", status_code=200) + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", + ) + print(f"Error installing job {release_name}: {helm_install_result.stderr.decode()}") + else: - err = True + print(f"Error listing Spark jobs: {helm_ls_result.stderr.decode()}") result = ActionResponse( status="ERROR", status_code=500, - error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", - ) - print( - f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + error_message="SPARK_HELM_LIST_EXCEPTION", ) - - if err is None: - result = ActionResponse(status="OK", status_code=200) + + if result is None: + result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") return result def _get_connector_details(self, dataset_id): active_connectors = [] + connector_versions = {} records = self.db_service.execute_select_all( f""" - SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology + SELECT ci.id, ci.connector_id, ci.operations_config, cr.runtime as connector_runtime, cr.source as connector_source, cr.technology, cr.version FROM connector_instances ci JOIN connector_registry cr on ci.connector_id = cr.id WHERE ci.status='{DatasetStatusType.Live.name}' and ci.dataset_id = '{dataset_id}' @@ -223,6 +349,7 @@ def _get_connector_details(self, dataset_id): active_connectors.append(from_dict( data_class=ConnectorInstance, data=record )) + # connector_versions[connector_instance.id] = record['version'] return active_connectors @@ -240,3 +367,38 @@ def _get_masterdata_details(self, dataset_id): is_masterdata = True return is_masterdata + + def _is_dataset_retired(self, dataset_id): + is_retired = False + rows = self.db_service.execute_select_all( + f""" + SELECT status + FROM datasets + WHERE status='{DatasetStatusType.Retired.name}' AND dataset_id = '{dataset_id}' + """ + ) + for row in rows: + if row['status'] == DatasetStatusType.Retired.name: + print("Status: ",row['status']) + is_retired = True + break + + return is_retired + + def _get_live_instances(self, dataset_id, runtime, connector_instance): + active_connectors = [] + records = self.db_service.execute_select_all( + f""" + SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id + FROM connector_instances ci + JOIN connector_registry cr ON ci.connector_id = cr.id + JOIN datasets d ON ci.dataset_id = d.id + WHERE cr.runtime = '{runtime}' AND ci.status = '{DatasetStatusType.Live.name}'; + """ + ) + + for record in records: + active_connectors.append(record['connector_instance_id']) + + return active_connectors + From ce5ccff713b09e1e0549165ceb1f1977e8fc03b1 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 20 Aug 2024 12:55:51 +0530 Subject: [PATCH 139/235] #OBS-I167 : read api changes while reading connectors according to v2 structure --- .../controllers/DatasetRead/DatasetRead.ts | 56 +++++++++++++++---- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 28909ff3..0752a013 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -39,36 +39,70 @@ const datasetRead = async (req: Request, res: Response) => { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } if (dataset.connectors_config) { - dataset.connectors_config = dataset.connectors_config.map((connector: any) => ({ - ...connector, - connector_config: JSON.parse(cipherService.decrypt(connector.connector_config)) - })); + dataset.connectors_config = dataset?.connectors_config.map((connector: any) => { + const connector_config = _.get(connector, "connector_config") + return { + ...connector, + connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector.connector_config)) + } + }); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { - + const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); - if(draftDataset) { // Contains a draft + if (draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset) - return _.pick(dataset, attributes); + return _.pick(dataset, attributes); } const liveDataset = await datasetService.getDataset(datasetId, undefined, true); - if(liveDataset) { + if (liveDataset) { const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) - return _.pick(dataset, attributes); + return _.pick(dataset, attributes); } - + return null; } const readDataset = async (datasetId: string, attributes: string[]): Promise => { const dataset = await datasetService.getDataset(datasetId, attributes, true); - return dataset; + const api_version = _.get(dataset, "api_version") + let datasetConfigs: any = {} + const connectors_config: any[] = await datasetService.getConnectorsV1(datasetId, ["id", "dataset_id", "connector_config", "connector_type"]); + const transformations_config = await datasetService.getTransformations(datasetId, ["field_key", "transformation_function", "mode", "metadata"]) + if (api_version !== "v2") { + datasetConfigs["connectors_config"] = _.map(connectors_config, (config) => { + return { + id: _.get(config, "id"), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), + version: "v1" + } + }) + datasetConfigs["transformations_config"] = _.map(transformations_config, (config) => { + console.log(config) + const section: any = _.get(config, "metadata.section") || _.get(config, "transformation_function.category"); + return { + field_key: _.get(config, "field_key"), + transformation_function: { + ..._.get(config, ["transformation_function"]), + datatype: _.get(config, "metadata._transformedFieldDataType") || _.get(config, "transformation_function.datatype") || "string", + category: datasetService.getTransformationCategory(section), + }, + mode: _.get(config, "mode") + } + }) + } + else { + datasetConfigs["connectors_config"] = connectors_config; + datasetConfigs["transformations_config"] = transformations_config; + } + return { ...dataset, ...datasetConfigs }; } export default datasetRead; \ No newline at end of file From 6da29abbfce53c8b9d63e9f91b0c0cf5ac63d743 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 13:16:39 +0530 Subject: [PATCH 140/235] #OBS-I173: fix: Ready to publish schema fix to expect connector configs as object and string --- .../DatasetStatusTransition/ReadyToPublishSchema.json | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index 281c62ac..f378a5f7 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -376,7 +376,14 @@ "minLength": 1 }, "connector_config": { - "type": "string" + "oneOf": [ + { + "type": "string" + }, + { + "type": "object" + } + ] }, "operations_config": { "type": "object" From 03565483c9dbe1781e9130788dbc5c0353fb546c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 13:18:19 +0530 Subject: [PATCH 141/235] #OBS-I174: fix: Dataset read api fix to expect both v1 and v2 connectors --- api-service/src/services/DatasetService.ts | 103 +++++++++++---------- 1 file changed, 56 insertions(+), 47 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 25b120f2..c0ccaf1d 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -25,7 +25,7 @@ class DatasetService { } findDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Dataset.findAll({where, attributes, order, raw: true}) + return Dataset.findAll({ where, attributes, order, raw: true }) } getDuplicateDenormKey = (denormConfig: Record): Array => { @@ -39,9 +39,9 @@ class DatasetService { } checkDatasetExists = async (dataset_id: string): Promise => { - const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes:["id"], raw: true }); + const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes: ["id"], raw: true }); if (draft === null) { - const live = await Dataset.findOne({ where: { id: dataset_id }, attributes:["id"], raw: true }); + const live = await Dataset.findOne({ where: { id: dataset_id }, attributes: ["id"], raw: true }); return !(live === null) } else { return true; @@ -53,7 +53,7 @@ class DatasetService { } findDraftDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return DatasetDraft.findAll({where, attributes, order, raw: true}) + return DatasetDraft.findAll({ where, attributes, order, raw: true }) } getDraftTransformations = async (dataset_id: string, attributes?: string[]) => { @@ -68,7 +68,7 @@ class DatasetService { return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); } - getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { + getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); } @@ -78,7 +78,7 @@ class DatasetService { updateDraftDataset = async (draftDataset: Record): Promise> => { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }}); + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } }); const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key }; logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }); return responseData; @@ -96,7 +96,7 @@ class DatasetService { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); const transaction = await sequelize.transaction(); - + try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); @@ -150,9 +150,9 @@ class DatasetService { return draftDataset; } - getTransformationCategory = (section: string):string => { + getTransformationCategory = (section: string): string => { - switch(section) { + switch (section) { case "pii": return "pii"; case "additionalFields": @@ -163,15 +163,15 @@ class DatasetService { } createDraftDatasetFromLive = async (dataset: Model) => { - - let draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); - const dataset_config:any = _.get(dataset, "dataset_config"); - const api_version:any = _.get(dataset, "api_version"); - if(api_version === "v1") { + + let draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); + const dataset_config: any = _.get(dataset, "dataset_config"); + const api_version: any = _.get(dataset, "api_version"); + if (api_version === "v1") { draftDataset["dataset_config"] = { - indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, - keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, - cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master") }, + keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, + cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } } const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { @@ -198,21 +198,30 @@ class DatasetService { draftDataset["sample_data"] = dataset_config?.mergedEvent draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) } else { - const connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); - draftDataset["connectors_config"] = connectors + const v1connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); + const modifiedV1Connectors = _.map(v1connectors, (config) => { + return { + id: _.get(config, "id"), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), + version: "v1" + } + }) + const v2connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); + draftDataset["connectors_config"] = _.concat(modifiedV1Connectors, v2connectors) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode"]); draftDataset["transformations_config"] = transformations } const denormConfig = _.get(draftDataset, "denorm_config") if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { - const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id","dataset_id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id", "dataset_id", "status", "dataset_config", "api_version"]) if (_.isEmpty(masterDatasets)) { throw { code: "DEPENDENT_MASTER_DATA_NOT_FOUND", message: `The dependent dataset not found`, errCode: "NOT_FOUND", statusCode: 404 } } const updatedDenormFields = _.map(denormConfig.denorm_fields, field => { const { redis_db, denorm_out_field, denorm_key } = field let masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.cache_config.redis_db") === redis_db) - if(!masterConfig){ + if (!masterConfig) { masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.redis_db") === redis_db) } if (_.isEmpty(masterConfig)) { @@ -238,12 +247,12 @@ class DatasetService { const { id } = dataset const transaction = await sequelize.transaction() try { - await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetDraft.destroy({ where: { id } , transaction}) + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasourceDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetDraft.destroy({ where: { id }, transaction }) await transaction.commit() - } catch (err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) } @@ -255,18 +264,18 @@ class DatasetService { try { await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); + await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); await this.deleteDruidSupervisors(dataset); - } catch(err:any) { + } catch (err: any) { await transaction.rollback(); throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); } } findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Datasource.findAll({where, attributes, order, raw: true}) + return Datasource.findAll({ where, attributes, order, raw: true }) } private deleteDruidSupervisors = async (dataset: Record) => { @@ -290,26 +299,26 @@ class DatasetService { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() try { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } , transaction}) - if(indexingConfig.olap_store_enabled) { + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }, transaction }) + if (indexingConfig.olap_store_enabled) { await this.createDruidDataSource(draftDataset, transaction); } - if(indexingConfig.lakehouse_enabled) { + if (indexingConfig.lakehouse_enabled) { const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id", "api_version"], true); - if(liveDataset && liveDataset.api_version === "v2") { + if (liveDataset && liveDataset.api_version === "v2") { await this.updateHudiDataSource(draftDataset, transaction) } else { await this.createHudiDataSource(draftDataset, transaction) } } await transaction.commit() - } catch(err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } await executeCommand(draftDataset.id, "PUBLISH_DATASET"); - + } private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -318,7 +327,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -327,25 +336,25 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); - const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") - const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record + const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") + const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } - private createDraftDatasource = (draftDataset: Record, type: string) : Record => { + private createDraftDatasource = (draftDataset: Record, type: string): Record => { - const datasource = _.join([draftDataset.dataset_id,"events"], "_") + const datasource = _.join([draftDataset.dataset_id, "events"], "_") return { - id: _.join([datasource,type], '_'), + id: _.join([datasource, type], '_'), datasource: draftDataset.dataset_id, dataset_id: draftDataset.dataset_id, datasource_ref: datasource, @@ -356,15 +365,15 @@ class DatasetService { } export const getLiveDatasetConfigs = async (dataset_id: string) => { - + let datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) - if(!_.isEmpty(transformations)){ + if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations } - if(!_.isEmpty(connectors)){ + if (!_.isEmpty(connectors)) { datasetRecord["connectors_config"] = connectors } return datasetRecord; From 67343d8af1d9eb6c8785836c2c3067021b5a9133 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 16:00:02 +0530 Subject: [PATCH 142/235] #OBS-I146: fix: Test case fix for read api --- .../controllers/DatasetRead/DatasetRead.ts | 12 +-- api-service/src/services/DatasetService.ts | 94 +++++++++---------- .../DatasetCreate/Fixtures.ts | 55 +++++++++++ .../DatasetRead/DatasetRead.spec.ts | 12 +-- .../DatasetUpdate/DatasetConnectors.spec.ts | 87 +++++++++++++++++ .../DatasetUpdate/Fixtures.ts | 16 ++++ 6 files changed, 217 insertions(+), 59 deletions(-) create mode 100644 api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index fc16b313..4265788d 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -48,21 +48,21 @@ const datasetRead = async (req: Request, res: Response) => { } const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { - + const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); - if(draftDataset) { // Contains a draft + if (draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset) - return _.pick(dataset, attributes); + return _.pick(dataset, attributes); } const liveDataset = await datasetService.getDataset(datasetId, undefined, true); - if(liveDataset) { + if (liveDataset) { const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) - return _.pick(dataset, attributes); + return _.pick(dataset, attributes); } - + return null; } diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index f5298126..50b42668 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -25,7 +25,7 @@ class DatasetService { } findDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Dataset.findAll({where, attributes, order, raw: true}) + return Dataset.findAll({ where, attributes, order, raw: true }) } getDuplicateDenormKey = (denormConfig: Record): Array => { @@ -39,9 +39,9 @@ class DatasetService { } checkDatasetExists = async (dataset_id: string): Promise => { - const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes:["id"], raw: true }); + const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes: ["id"], raw: true }); if (draft === null) { - const live = await Dataset.findOne({ where: { id: dataset_id }, attributes:["id"], raw: true }); + const live = await Dataset.findOne({ where: { id: dataset_id }, attributes: ["id"], raw: true }); return !(live === null) } else { return true; @@ -53,7 +53,7 @@ class DatasetService { } findDraftDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return DatasetDraft.findAll({where, attributes, order, raw: true}) + return DatasetDraft.findAll({ where, attributes, order, raw: true }) } getDraftTransformations = async (dataset_id: string, attributes?: string[]) => { @@ -68,7 +68,7 @@ class DatasetService { return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); } - getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { + getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); } @@ -78,7 +78,7 @@ class DatasetService { updateDraftDataset = async (draftDataset: Record): Promise> => { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }}); + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } }); const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key }; logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }); return responseData; @@ -96,7 +96,7 @@ class DatasetService { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); const transaction = await sequelize.transaction(); - + try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); @@ -150,9 +150,9 @@ class DatasetService { return draftDataset; } - getTransformationCategory = (section: string):string => { + getTransformationCategory = (section: string): string => { - switch(section) { + switch (section) { case "pii": return "pii"; case "additionalFields": @@ -163,15 +163,15 @@ class DatasetService { } createDraftDatasetFromLive = async (dataset: Model) => { - - const draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); - const dataset_config:any = _.get(dataset, "dataset_config"); - const api_version:any = _.get(dataset, "api_version"); - if(api_version === "v1") { + + const draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); + const dataset_config: any = _.get(dataset, "dataset_config"); + const api_version: any = _.get(dataset, "api_version"); + if (api_version === "v1") { draftDataset["dataset_config"] = { - indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, - keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, - cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master") }, + keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, + cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } } const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { @@ -205,14 +205,14 @@ class DatasetService { } const denormConfig = _.get(draftDataset, "denorm_config") if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { - const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id","dataset_id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id", "dataset_id", "status", "dataset_config", "api_version"]) if (_.isEmpty(masterDatasets)) { throw { code: "DEPENDENT_MASTER_DATA_NOT_FOUND", message: `The dependent dataset not found`, errCode: "NOT_FOUND", statusCode: 404 } } const updatedDenormFields = _.map(denormConfig.denorm_fields, field => { const { redis_db, denorm_out_field, denorm_key } = field let masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.cache_config.redis_db") === redis_db) - if(!masterConfig){ + if (!masterConfig) { masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.redis_db") === redis_db) } if (_.isEmpty(masterConfig)) { @@ -225,8 +225,8 @@ class DatasetService { draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft - await DatasetDraft.create(draftDataset); - return await this.getDraftDataset(draftDataset.dataset_id); + const result = await DatasetDraft.create(draftDataset); + return _.get(result, "dataValues") } getNextRedisDBIndex = async () => { @@ -238,12 +238,12 @@ class DatasetService { const { id } = dataset const transaction = await sequelize.transaction() try { - await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetDraft.destroy({ where: { id } , transaction}) + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasourceDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetDraft.destroy({ where: { id }, transaction }) await transaction.commit() - } catch (err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) } @@ -255,18 +255,18 @@ class DatasetService { try { await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); + await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); await this.deleteDruidSupervisors(dataset); - } catch(err:any) { + } catch (err: any) { await transaction.rollback(); throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); } } findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Datasource.findAll({where, attributes, order, raw: true}) + return Datasource.findAll({ where, attributes, order, raw: true }) } private deleteDruidSupervisors = async (dataset: Record) => { @@ -290,26 +290,26 @@ class DatasetService { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() try { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } , transaction}) - if(indexingConfig.olap_store_enabled) { + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }, transaction }) + if (indexingConfig.olap_store_enabled) { await this.createDruidDataSource(draftDataset, transaction); } - if(indexingConfig.lakehouse_enabled) { + if (indexingConfig.lakehouse_enabled) { const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id", "api_version"], true); - if(liveDataset && liveDataset.api_version === "v2") { + if (liveDataset && liveDataset.api_version === "v2") { await this.updateHudiDataSource(draftDataset, transaction) } else { await this.createHudiDataSource(draftDataset, transaction) } } await transaction.commit() - } catch(err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } await executeCommand(draftDataset.id, "PUBLISH_DATASET"); - + } private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -318,7 +318,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -327,25 +327,25 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); - const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") - const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record + const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") + const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } - private createDraftDatasource = (draftDataset: Record, type: string) : Record => { + private createDraftDatasource = (draftDataset: Record, type: string): Record => { - const datasource = _.join([draftDataset.dataset_id,"events"], "_") + const datasource = _.join([draftDataset.dataset_id, "events"], "_") return { - id: _.join([datasource,type], "_"), + id: _.join([datasource, type], "_"), datasource: draftDataset.dataset_id, dataset_id: draftDataset.dataset_id, datasource_ref: datasource, @@ -356,15 +356,15 @@ class DatasetService { } export const getLiveDatasetConfigs = async (dataset_id: string) => { - + const datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) - if(!_.isEmpty(transformations)){ + if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations } - if(!_.isEmpty(connectors)){ + if (!_.isEmpty(connectors)) { datasetRecord["connectors_config"] = connectors } return datasetRecord; diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts index 46aaebeb..01b87ac1 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/Fixtures.ts @@ -161,6 +161,54 @@ export const TestInputsForDatasetCreate = { } }, + VALID_DATASET_WITH_CONNECTORS: { + "id": "api.datasets.create", + "ver": "v1", + "ts": "2024-04-10T16:10:50+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, + "request": { + "dataset_id": "sb-ddd", + "type": "event", + "name": "sb-telemetry2", + "data_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "eid": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "ets": { + "type": "string" + }, + "required": [ + "eid" + ] + }, + "additionalProperties": true + }, + "dataset_config": { + "indexing_config": { + "olap_store_enabled": false, + "lakehouse_enabled": true, + "cache_enabled": false + }, + "keys_config": { + "timestamp_key": "ets" + }, + "file_upload_path": [ + "telemetry.json" + ] + }, + "connectors_config":[{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}, {"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5cai","connector_id":"debezium","connector_config":{"type":"debezium","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}], + "tags": [] + } + }, + VALID_MINIMAL_DATASET: { "id": "api.datasets.create", "ver": "v2", @@ -536,6 +584,13 @@ export const DATASET_CREATE_SUCCESS_FIXTURES = [ "status": "SUCCESS", "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" }, + { + "title": "Dataset creation success: When connectors payload provided", + "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_CONNECTORS, + "httpStatus": httpStatus.OK, + "status": "SUCCESS", + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6d" + }, { "title": "Dataset creation success: When multiple transformation payload provided with same field key", "requestPayload": TestInputsForDatasetCreate.VALID_DATASET_WITH_MULTIPLE_TRANSFORMATIONS, diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 8b104ec9..ca0ec0d1 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -25,7 +25,7 @@ describe("DATASET READ API", () => { it("Dataset read success: When minimal fields requested", (done) => { chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ "name": "sb-telemetry", "data_version": 1 }) + return Promise.resolve({ "name": "sb-telemetry", "version": 1 }) }) chai .request(app) @@ -38,7 +38,7 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ name: "sb-telemetry", data_version: 1 })) + result.should.be.eq(JSON.stringify({ name: "sb-telemetry", version: 1 })) done(); }); }); @@ -64,7 +64,7 @@ describe("DATASET READ API", () => { }); }); - it("Dataset read success: Fetch live dataset when status param is empty", (done) => { + it("Dataset read success: Fetch live dataset when mode param not provided", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve(TestInputsForDatasetRead.LIVE_SCHEMA) }) @@ -150,20 +150,20 @@ describe("DATASET READ API", () => { it("Dataset read failure: Updating dataset status to draft on mode=edit fails as live record not found", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ dataset_id: "sb-telemetry", name: "sb-telemetry", status: "Live", data_schema: {} }) + return Promise.resolve() }) chai.spy.on(Dataset, "findOne", () => { return Promise.resolve() }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?status=Draft&mode=edit") + .get("/v2/datasets/read/sb-telemetry?mode=edit") .end((err, res) => { res.should.have.status(httpStatus.NOT_FOUND); res.body.should.be.a("object") res.body.id.should.be.eq(apiId); res.body.params.status.should.be.eq("FAILED") - res.body.error.message.should.be.eq("Failed to fetch live dataset") + res.body.error.message.should.be.eq("Dataset with the given dataset_id:sb-telemetry not found") res.body.error.code.should.be.eq("DATASET_NOT_FOUND") done(); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts new file mode 100644 index 00000000..34907197 --- /dev/null +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts @@ -0,0 +1,87 @@ +import app from "../../../app"; +import chai from "chai"; +import chaiHttp from "chai-http"; +import spies from "chai-spies"; +import httpStatus from "http-status"; +import { describe, it } from "mocha"; +import { DatasetDraft } from "../../../models/DatasetDraft"; +import _ from "lodash"; +import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; +import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" +import { sequelize } from "../../../connections/databaseConnection"; + +chai.use(spies); +chai.should(); +chai.use(chaiHttp); + +describe("DATASET CONNECTORS UPDATE", () => { + + afterEach(() => { + chai.spy.restore(); + }); + + it("Success: Dataset connectors successfully added", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", connectors_config:[] + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_CONNECTORS_ADD) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + + it("Success: Dataset connectors successfully removed", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", status: "Draft", version_key: validVersionKey, type:"event", api_version: "v2", connectors_config:[{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}] + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_CONNECTORS_REMOVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + +}) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index eb2c683b..97248d9f 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -88,6 +88,22 @@ export const TestInputsForDatasetUpdate = { } }, + DATASET_UPDATE_CONNECTORS_ADD: { + ...requestStructure, request: { + "dataset_id": "telemetry", + "version_key": validVersionKey, + "connectors_config":[{"value":{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}, "action": "upsert"}], + } + }, + + DATASET_UPDATE_CONNECTORS_REMOVE: { + ...requestStructure, request: { + "dataset_id": "telemetry", + "version_key": validVersionKey, + "connectors_config":[{"value":{"id":"6c3fc8c2-357d-489b-b0c9-afdde6e5c6c0","connector_id":"kafka","connector_config":{"type":"kafka","topic":"telemetry.ingest","kafkaBrokers":"kafka-headless.kafka.svc:9092"},"version":"v1"}, "action": "upsert"}], + } + }, + DATASET_UPDATE_DEDUP_DUPLICATES_TRUE: { ...requestStructure, request: { From 680f67f88e9e170119de965946dd8bfc2fd89c36 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 18:23:03 +0530 Subject: [PATCH 143/235] #OBS-I146: fix: Test case fix for read api changes --- .../controllers/DatasetRead/DatasetRead.ts | 9 +-- .../DatasetRead/DatasetRead.spec.ts | 52 ++++++++++++- .../DatasetManagement/DatasetRead/Fixtures.ts | 78 +++++++++++++------ .../DatasetReadyToPublish.spec.ts | 13 +--- .../DatasetUpdate/DatasetConnectors.spec.ts | 13 ---- .../DatasetUpdate/DatasetDedup.spec.ts | 13 ---- .../DatasetUpdate/DatasetDenorm.spec.ts | 19 ----- .../DatasetUpdate/DatasetExtraction.spec.ts | 14 +--- .../DatasetUpdate/DatasetTags.spec.ts | 14 +--- .../DatasetTransformation.spec.ts | 21 +---- .../DatasetUpdate/DatasetUpdate.spec.ts | 14 +--- .../DatasetUpdate/DatasetValidation.spec.ts | 14 +--- 12 files changed, 119 insertions(+), 155 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 4265788d..2822e87b 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -11,16 +11,13 @@ export const apiId = "api.datasets.read"; export const errorCode = "DATASET_READ_FAILURE" // TODO: Move this to a config -const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config"] +export const defaultFields = ["dataset_id", "name", "type", "status", "tags", "version", "api_version", "dataset_config"] const validateRequest = (req: Request) => { const { dataset_id } = req.params; - const fields = req.query.fields; - if (fields && typeof fields !== "string") { - throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS_VAL", `The specified fields [${fields}] in the query param is not a string.`, "BAD_REQUEST", 400); - } - const fieldValues = fields ? _.split(fields, ",") : []; + const { fields } = req.query; + const fieldValues = fields ? _.split(fields as string, ",") : []; const invalidFields = _.difference(fieldValues, Object.keys(DatasetDraft.getAttributes())); if (!_.isEmpty(invalidFields)) { throw obsrvError(dataset_id, "DATASET_INVALID_FIELDS", `The specified fields [${invalidFields}] in the dataset cannot be found.`, "BAD_REQUEST", 400); diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index ca0ec0d1..2c05ce87 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -5,13 +5,17 @@ import spies from "chai-spies"; import httpStatus from "http-status"; import { describe, it } from "mocha"; import _ from "lodash"; -import { apiId } from "../../../controllers/DatasetRead/DatasetRead"; +import { apiId, defaultFields } from "../../../controllers/DatasetRead/DatasetRead"; import { TestInputsForDatasetRead } from "./Fixtures"; import { DatasetTransformations } from "../../../models/Transformation"; import { Dataset } from "../../../models/Dataset"; import { DatasetDraft } from "../../../models/DatasetDraft"; import { DatasetSourceConfig } from "../../../models/DatasetSourceConfig"; import { ConnectorInstances } from "../../../models/ConnectorInstances"; +import { DatasetTransformationsDraft } from "../../../models/TransformationDraft"; +import { DatasetSourceConfigDraft } from "../../../models/DatasetSourceConfigDraft"; +import { sequelize } from "../../../connections/databaseConnection"; +import { DatasourceDraft } from "../../../models/DatasourceDraft"; chai.use(spies); chai.should(); @@ -134,7 +138,7 @@ describe("DATASET READ API", () => { }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?status=Draft&mode=edit") + .get("/v2/datasets/read/sb-telemetry?mode=edit") .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -148,6 +152,50 @@ describe("DATASET READ API", () => { }); }); + it("Dataset read success: Migrating v1 draft dataset to v2 on mode=edit", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetRead.DRAFT_SCHEMA_V1) + }) + chai.spy.on(DatasetTransformationsDraft, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfigDraft, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) + }) + chai.spy.on(DatasetTransformationsDraft, "destroy", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasetSourceConfigDraft, "destroy", () => { + return Promise.resolve({}) + }) + chai.spy.on(DatasourceDraft, "destroy", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.result.name.should.be.eq("sb-telemetry") + const result = JSON.stringify(res.body.result) + result.should.be.eq(JSON.stringify(_.pick(TestInputsForDatasetRead.DRAFT_SCHEMA_V1, defaultFields))) + done(); + }); + }); + it("Dataset read failure: Updating dataset status to draft on mode=edit fails as live record not found", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve() diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts index 61760c6e..c1ecd1e2 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -24,6 +24,49 @@ export const TestInputsForDatasetRead = { ] } }, + DRAFT_SCHEMA_V1: { + "dataset_id": "sb-telemetry", + "name": "sb-telemetry", + "type": "event", + "status": "Draft", + "tags": [ + "tag1", + "tag2" + ], + "validation_config": { + "validate": true, + "validation_mode": "Strict", + "mode": "Strict" + }, + "data_schema": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "eid": { + "type": "string" + }, + "ver": { + "type": "string" + }, + "ets": { + "type": "string" + }, + "required": [ + "eid" + ] + }, + "additionalProperties": true + }, + "version": 1, + "api_version": "v1", + "dataset_config": { + "timestamp_key": "ets", + "data_key": "", + "redis_db_host": "localhost", + "redis_db_port": 6379, + "redis_db": 0 + } + }, LIVE_SCHEMA: { "dataset_id": "sb-telemetry", "name": "sb-telemetry", @@ -49,7 +92,7 @@ export const TestInputsForDatasetRead = { ] } }, - TRANSFORMATIONS_SCHEMA:[{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], + TRANSFORMATIONS_SCHEMA: [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], TRANSFORMATIONS_SCHEMA_V1: [ { "field_key": "eid", @@ -67,27 +110,14 @@ export const TestInputsForDatasetRead = { } } ], - DATASOURCE_SCHEMA: { - "id": "sb-telemetry_sb-telemetry", - "datasource": "sb-telemetry", - "dataset_id": "sb-telemetry", - "ingestion_spec": { "type": "kafka", "spec": { "dataSchema": { "dataSource": "dataset-conf_day", "dimensionsSpec": { "dimensions": [{ "type": "string", "name": "a" }, { "type": "string", "name": "obsrv.meta.source.connector" }, { "type": "string", "name": "obsrv.meta.source.id" }] }, "timestampSpec": { "column": "obsrv_meta.syncts", "format": "auto" }, "metricsSpec": [], "granularitySpec": { "type": "uniform", "segmentGranularity": "DAY", "queryGranularity": "none", "rollup": false } }, "tuningConfig": { "type": "kafka", "maxBytesInMemory": 134217728, "maxRowsPerSegment": 5000000, "logParseExceptions": true }, "ioConfig": { "type": "kafka", "consumerProperties": { "bootstrap.servers": "localhost:9092" }, "taskCount": 1, "replicas": 1, "taskDuration": "PT1H", "useEarliestOffset": true, "completionTimeout": "PT1H", "inputFormat": { "type": "json", "flattenSpec": { "useFieldDiscovery": true, "fields": [{ "type": "path", "expr": "$.['a']", "name": "a" }, { "type": "path", "expr": "$.obsrv_meta.['syncts']", "name": "obsrv_meta.syncts" }, { "type": "path", "expr": "$.obsrv_meta.source.['connector']", "name": "obsrv.meta.source.connector" }, { "type": "path", "expr": "$.obsrv_meta.source.['connectorInstance']", "name": "obsrv.meta.source.id" }, { "expr": "$.obsrv_meta.syncts", "name": "obsrv_meta.syncts", "type": "path" }] } }, "appendToExisting": false } } }, - "datasource_ref": "sb-telemetry_DAY", - "retention_period": { - "enabled": "false" - }, - "archival_policy": { - "enabled": "false" - }, - "purge_policy": { - "enabled": "false" - }, - "backup_config": { - "enabled": "false" - }, - "status": "Live", - "created_by": "SYSTEM", - "updated_by": "SYSTEM", - "published_date": "2023-07-03 00:00:00" - } + CONNECTORS_SCHEMA_V1: [ + { + "id": "hsh882ehdshe", + "connector_type": "kafka", + "connector_config": { + "topic": "local.ingest", + "brokerURL": "localhost:9092" + } + } + ] } \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index c82e809d..2ce2fdfd 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -29,12 +29,7 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({}) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .post("/v2/datasets/status-transition") @@ -97,12 +92,6 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(TestInputsForDatasetStatusTransition.INVALID_SCHEMA_FOR_READY_TO_PUBLISH) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "rollback", () => { - return Promise.resolve({}) - }) chai .request(app) .post("/v2/datasets/status-transition") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts index 34907197..1122aac4 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetConnectors.spec.ts @@ -8,7 +8,6 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); chai.should(); @@ -29,12 +28,6 @@ describe("DATASET CONNECTORS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +54,6 @@ describe("DATASET CONNECTORS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts index e8f614c8..eaa80648 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDedup.spec.ts @@ -8,7 +8,6 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, requestStructure, validVersionKey } from "./Fixtures"; import { apiId, invalidInputErrCode } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); chai.should(); @@ -29,12 +28,6 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +54,6 @@ describe("DATASET DEDUPE CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index cdf02c4b..afc673ca 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -8,7 +8,6 @@ import { DatasetDraft } from "../../../models/DatasetDraft"; import _ from "lodash"; import { TestInputsForDatasetUpdate, msgid, validVersionKey } from "./Fixtures"; import { apiId } from "../../../controllers/DatasetUpdate/DatasetUpdate" -import { sequelize } from "../../../connections/databaseConnection"; chai.use(spies); chai.should(); @@ -29,12 +28,6 @@ describe("DATASET DENORM UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +54,6 @@ describe("DATASET DENORM UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -99,12 +86,6 @@ describe("DATASET DENORM UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts index 24ccc831..d17f1c3b 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetExtraction.spec.ts @@ -29,12 +29,7 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +56,7 @@ describe("DATASET EXTRACTION CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts index fe3715d4..49ba7b8a 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTags.spec.ts @@ -29,12 +29,7 @@ describe("DATASET TAGS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +56,7 @@ describe("DATASET TAGS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts index 0ae345ff..58252e83 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetTransformation.spec.ts @@ -29,12 +29,7 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +56,7 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -91,12 +81,7 @@ describe("DATASET TRANSFORMATIONS UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index 1818bc79..6cb84e25 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -29,12 +29,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -66,12 +61,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts index e77ab8b0..d5feed17 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetValidation.spec.ts @@ -29,12 +29,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") @@ -61,12 +56,7 @@ describe("DATASET VALIDATION CONFIG UPDATE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) + chai .request(app) .patch("/v2/datasets/update") From 13c499e57a1eefb9f498a40f98dbf17f592adaea Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 23:46:50 +0530 Subject: [PATCH 144/235] #OBS-I146: fix: status transition test cases --- .../DatasetCreate/DatasetCreate.spec.ts | 3 - .../DatasetRead/DatasetRead.spec.ts | 4 +- .../DatasetManagement/DatasetRead/Fixtures.ts | 10 ++ .../DatasetLive.spec.ts | 109 ++++++++++++++++++ .../DatasetReadyToPublish.spec.ts | 29 ++++- .../DatasetStatusTransition/Fixtures.ts | 42 ++++++- 6 files changed, 188 insertions(+), 9 deletions(-) diff --git a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts index a0e2ace6..51953ba0 100644 --- a/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetCreate/DatasetCreate.spec.ts @@ -30,9 +30,6 @@ describe("DATASET CREATE API", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve(null) }) - chai.spy.on(sequelize, "query", () => { - return Promise.resolve([{ nextVal: 9 }]) - }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: { id: "telemetry" } }) }) diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 2c05ce87..6191d43e 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -99,7 +99,7 @@ describe("DATASET READ API", () => { return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) }) chai.spy.on(ConnectorInstances, "findAll", () => { - return Promise.resolve([]) + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) @@ -131,7 +131,7 @@ describe("DATASET READ API", () => { return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) }) chai.spy.on(DatasetSourceConfig, "findAll", () => { - return Promise.resolve([]) + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts index c1ecd1e2..2efaec73 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -119,5 +119,15 @@ export const TestInputsForDatasetRead = { "brokerURL": "localhost:9092" } } + ], + CONNECTORS_SCHEMA_V2: [ + { + "id": "hsh882ehdshe", + "connector_id": "kafka", + "connector_config": { + "topic": "local.ingest", + "brokerURL": "localhost:9092" + } + } ] } \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index bb0c8e51..cd32573f 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -66,6 +66,115 @@ describe("DATASET STATUS TRANSITION LIVE", () => { }); }); + + it("Dataset status transition success: When the action is to set master dataset live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_SCHEMA_FOR_PUBLISH) + }) + chai.spy.on(sequelize, "query", () => { + return Promise.resolve([[{ nextval: 9 }]]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("master-telemetry") + done(); + }); + }); + + it("Dataset status transition failure: When the dependent denorm master dataset is not live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(_.clone(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH)) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Retired", "dataset_config": { "redis_db": 21 }, "api_version": "v1" }]) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.PRECONDITION_REQUIRED); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("The datasets with id:master-dataset are not in published status") + res.body.error.code.should.be.eq("DEPENDENT_MASTER_DATA_NOT_LIVE") + done(); + }); + }); + + it("Dataset status transition failure: When dataset to publish is self referencing the denorm master dataset", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({...TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH, "id": "master-dataset"}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.CONFLICT); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("The denorm master dataset is self-referencing itself") + res.body.error.code.should.be.eq("SELF_REFERENCING_MASTER_DATA") + done(); + }); + }); + + it("Dataset status transition failure: Unable to fetch redis db number for master dataset", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_INVALID) + }) + chai.spy.on(sequelize, "query", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) + .end((err, res) => { + res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Unable to fetch the redis db index for the master data") + res.body.error.code.should.be.eq("REDIS_DB_INDEX_FETCH_FAILED") + done(); + }); + }); + it("Dataset status transition failure: When dataset is not found to publish", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve() diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts index 2ce2fdfd..f7d5f1f3 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetReadyToPublish.spec.ts @@ -7,8 +7,6 @@ import { describe, it } from "mocha"; import _ from "lodash"; import { TestInputsForDatasetStatusTransition } from "./Fixtures"; import { DatasetDraft } from "../../../models/DatasetDraft"; -import { sequelize } from "../../../connections/databaseConnection"; - chai.use(spies); chai.should(); @@ -29,7 +27,32 @@ describe("DATASET STATUS TRANSITION READY TO PUBLISH", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({}) }) - + + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_REQUEST_FOR_READY_FOR_PUBLISH) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to ReadyToPublish successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition success: When the action is make master dataset ready to publish", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.VALID_MASTER_SCHEMA_FOR_READY_TO_PUBLISH) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai .request(app) .post("/v2/datasets/status-transition") diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index 9a807e7f..ecf531f0 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -23,6 +23,18 @@ export const TestInputsForDatasetStatusTransition = { "status": "Live" } }, + VALID_SCHEMA_FOR_LIVE_MASTER: { + "id": "api.datasets.status-transition", + "ver": "v2", + "ts": "2024-04-19T12:58:47+05:30", + "params": { + "msgid": "4a7f14c3-d61e-4d4f-be78-181834eeff6" + }, + "request": { + "dataset_id": "master-telemetry", + "status": "Live" + } + }, VALID_SCHEMA_FOR_RETIRE: { "id": "api.datasets.status-transition", "ver": "v2", @@ -85,6 +97,32 @@ export const TestInputsForDatasetStatusTransition = { "sample_data": {}, "entry_topic": "local.ingest" }, + VALID_MASTER_SCHEMA_FOR_READY_TO_PUBLISH: { + "id": "dataset-all-fields7", + "dataset_id": "dataset-all-fields7", + "version": 1, + "type": "master", + "name": "sb-telemetry", + "validation_config": { "validate": false, "mode": "Strict" }, + "extraction_config": { "is_batch_event": true, "extraction_key": "events", "dedup_config": { "drop_duplicates": true, "dedup_key": "id", "dedup_period": 604800 } }, + "dedup_config": { "drop_duplicates": true, "dedup_key": "mid", "dedup_period": 604800 }, + "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "mid": { "type": "string", "arrival_format": "text", "data_type": "string" }, "ets": { "type": "integer", "arrival_format": "number", "data_type": "epoch" }, "eid": { "type": "string", "arrival_format": "text", "data_type": "string" } }, "additionalProperties": true }, + "denorm_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "denorm_fields": [{ "denorm_key": "eid", "denorm_out_field": "userdata", "dataset_id": "master-dataset", "redis_db": 85 }] }, + "router_config": { "topic": "dataset-all-fields7" }, + "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": true, "cache_enabled": false }, "keys_config": { "data_key": "eid", "partition_key": "eid", "timestamp_key": "obsrv_meta.syncts" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": [] }, + "tags": ["tag1"], + "status": "Draft", + "created_by": "SYSTEM", + "updated_by": "SYSTEM", + "created_date": "2024-07-24 19:12:13.021", + "updated_date": "2024-07-25 06:12:38.412", + "version_key": "1721887933020", + "transformations_config": [], + "connectors_config": [], + "api_version": "v2", + "sample_data": {}, + "entry_topic": "local.ingest" + }, INVALID_SCHEMA_FOR_READY_TO_PUBLISH: { "id": "dataset-all-fields7", "dataset_id": "dataset-all-fields7", @@ -119,5 +157,7 @@ export const TestInputsForDatasetStatusTransition = { "version_key": "1721887933020", "api_version": "v2" }, - DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } + DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, + DRAFT_MASTER_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "master-telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, + DRAFT_MASTER_DATASET_INVALID: { "dataset_id": "master-telemetry", "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } } \ No newline at end of file From 9c8d4ac7d319cfba6e5918af7de19275722fe6f9 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 23:49:29 +0530 Subject: [PATCH 145/235] #OBS-I146: fix: Test case script fix --- api-service/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/package.json b/api-service/package.json index 1360ecf6..1d49c029 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -6,7 +6,7 @@ "scripts": { "start": "ts-node ./src/app.ts", "test": "source .env.test && nyc mocha ./src/tests/**/*.spec.ts --exit", - "actions:test": "nyc mocha ./src/test/*.spec.ts --exit", + "actions:test": "nyc mocha ./src/tests/**/*.spec.ts --exit", "build": "rm -rf dist && tsc --declaration -P . && cp package.json ./dist/package.json", "package": "npm run build && cd dist && npm pack . && cd ..", "lint": "eslint . --ext .ts", From 2079680dad5b4757f171d5931e7d4fe6fb0a1f2a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 20 Aug 2024 23:54:58 +0530 Subject: [PATCH 146/235] #OBS-I146: fix: Type error fix --- api-service/src/services/CloudServices/AWSStorageService.ts | 2 +- api-service/src/services/CloudServices/AzureStorageService.ts | 2 +- api-service/src/services/CloudServices/GCPStorageService.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/services/CloudServices/AWSStorageService.ts b/api-service/src/services/CloudServices/AWSStorageService.ts index e538298f..d83b65b1 100644 --- a/api-service/src/services/CloudServices/AWSStorageService.ts +++ b/api-service/src/services/CloudServices/AWSStorageService.ts @@ -67,7 +67,7 @@ export class AWSStorageService implements ICloudService { async getSignedUrls(container: any, filesList: any) { const signedUrlsPromises = this.generateSignedURLs(container, filesList) const signedUrlsList = await Promise.all(signedUrlsPromises); - const periodWiseFiles: any = {}; + const periodWiseFiles: { [key: string]: string[] } = {}; const files: any[] = []; // Formatting response signedUrlsList.map(async (fileObject) => { diff --git a/api-service/src/services/CloudServices/AzureStorageService.ts b/api-service/src/services/CloudServices/AzureStorageService.ts index c90af51e..b4472aac 100644 --- a/api-service/src/services/CloudServices/AzureStorageService.ts +++ b/api-service/src/services/CloudServices/AzureStorageService.ts @@ -130,7 +130,7 @@ export class AzureStorageService implements ICloudService { const signedUrlsPromises = this.generateSignedURLs(container, filesList) const signedUrlsList = await Promise.all(signedUrlsPromises); const files: any[] = [] - const periodWiseFiles: any = {}; + const periodWiseFiles: { [key: string]: string[] } = {}; // Formatting response signedUrlsList.map(async (fileObject) => { const fileDetails = _.keys(fileObject); diff --git a/api-service/src/services/CloudServices/GCPStorageService.ts b/api-service/src/services/CloudServices/GCPStorageService.ts index 90fcaeb0..b07e3e4e 100644 --- a/api-service/src/services/CloudServices/GCPStorageService.ts +++ b/api-service/src/services/CloudServices/GCPStorageService.ts @@ -74,7 +74,7 @@ export class GCPStorageService implements ICloudService { return generateSignedUrls() .then(signedUrlList => { - const periodWiseFiles: any = {}; + const periodWiseFiles: { [key: string]: string[] } = {}; const files: any = []; const signedUrls = _.flattenDeep(_.map(signedUrlList, url => { const values = _.values(url) From 55737187add2a69245fdfa8176ac9b50953f4d3b Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 21 Aug 2024 10:33:49 +0530 Subject: [PATCH 147/235] #OBS-I141: removed metric for sum of response time --- .../DatasetCopy/DatasetCopyHelper.ts | 1 - .../src/controllers/DatasetRead/DatasetRead.ts | 17 +++++++++++++---- api-service/src/services/DatasetService.ts | 16 ++++++++++++---- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts index de4c8087..82b35ebd 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -10,7 +10,6 @@ export const updateRecords = (datasetRecord: Record, newDatasetId: _.set(datasetRecord, 'status', DatasetStatus.Draft) _.set(datasetRecord, "dataset_id", dataset_id) _.set(datasetRecord, "id", dataset_id) - _.set(datasetRecord, "name", dataset_id) _.set(datasetRecord, "version_key", Date.now().toString()) _.set(datasetRecord, 'version', version); _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 28909ff3..db2dc32f 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -39,10 +39,19 @@ const datasetRead = async (req: Request, res: Response) => { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } if (dataset.connectors_config) { - dataset.connectors_config = dataset.connectors_config.map((connector: any) => ({ - ...connector, - connector_config: JSON.parse(cipherService.decrypt(connector.connector_config)) - })); + dataset.connectors_config = dataset?.connectors_config.map((connector: any) => { + let connector_config = _.get(connector, "connector_config") + const authMechanism = _.get(connector_config, ["authenticationMechanism"]) + if (authMechanism && authMechanism.encrypted) { + connector_config = { + ...connector_config, + authenticationMechanism: JSON.parse(cipherService.decrypt(authMechanism.encryptedValues))} + } + return { + ...connector, + connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector_config)) + } + }); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 25b120f2..4f39b5d1 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -125,12 +125,14 @@ class DatasetService { const transformationFields = ["field_key", "transformation_function", "mode", "metadata"] const transformations = _.includes([DatasetStatus.Live], status) ? await this.getTransformations(dataset_id, transformationFields) : await this.getDraftTransformations(dataset_id, transformationFields); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, ["field_key"]), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section) }, mode: _.get(config, ["mode"]) } @@ -147,6 +149,7 @@ class DatasetService { }) draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) draftDataset["sample_data"] = dataset_config?.mergedEvent + draftDataset["status"] = DatasetStatus.Draft return draftDataset; } @@ -157,6 +160,8 @@ class DatasetService { return "pii"; case "additionalFields": return "derived"; + case "derived": + return "derived"; default: return "transform"; } @@ -184,12 +189,14 @@ class DatasetService { }) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, "field_key"), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, "metadata._transformedFieldDataType") || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section), }, mode: _.get(config, "mode") } @@ -225,8 +232,9 @@ class DatasetService { draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft - await DatasetDraft.create(draftDataset); - return await this.getDraftDataset(draftDataset.dataset_id); + const response = await DatasetDraft.create(draftDataset); + return _.get(response,"dataValues"); + // return await this.getDraftDataset(draftDataset.dataset_id); } getNextRedisDBIndex = async () => { From 4429b3362d6281e30d898b70567de1c4add3f83d Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 21 Aug 2024 11:01:33 +0530 Subject: [PATCH 148/235] #OBS-I146: fix: Dataset read api test cases fixes --- api-service/src/services/DatasourceService.ts | 338 ------------------ .../DatasetRead/DatasetRead.spec.ts | 66 ++++ .../DatasetManagement/DatasetRead/Fixtures.ts | 10 + .../DatasetLive.spec.ts | 45 ++- 4 files changed, 98 insertions(+), 361 deletions(-) diff --git a/api-service/src/services/DatasourceService.ts b/api-service/src/services/DatasourceService.ts index 96c8f294..f9f6b2a7 100644 --- a/api-service/src/services/DatasourceService.ts +++ b/api-service/src/services/DatasourceService.ts @@ -1,37 +1,5 @@ import _ from "lodash"; -import { ingestionConfig } from "../configs/IngestionConfig"; import { Datasource } from "../models/Datasource"; -import { DatasourceDraft } from "../models/DatasourceDraft"; -import { DatasetTransformationsDraft } from "../models/TransformationDraft"; -import { DatasetTransformations } from "../models/Transformation"; -import { DatasetStatus } from "../types/DatasetModels"; -import { Dataset } from "../models/Dataset"; - -export const DEFAULT_TIMESTAMP = { - indexValue: "obsrv_meta.syncts", - rootPath: "obsrv_meta", - label: "syncts", - path: "obsrv_meta.properties.syncts", -} - -const defaultTsObject = [ - { - "column": DEFAULT_TIMESTAMP.rootPath, - "type": "object", - "key": `properties.${DEFAULT_TIMESTAMP.rootPath}`, - "ref": `properties.${DEFAULT_TIMESTAMP.rootPath}`, - "isModified": true, - "required": false, - }, - { - "column": DEFAULT_TIMESTAMP.label, - "type": "integer", - "key": `properties.${DEFAULT_TIMESTAMP.path}`, - "ref": `properties.${DEFAULT_TIMESTAMP.path}`, - "isModified": true, - "required": false, - } -] export const getDatasourceList = async (datasetId: string, raw = false) => { const dataSource = await Datasource.findAll({ @@ -43,320 +11,14 @@ export const getDatasourceList = async (datasetId: string, raw = false) => { return dataSource } -export const getDraftDatasourceList = async (datasetId: string, raw = false) => { - const dataSource = await DatasourceDraft.findAll({ - where: { - dataset_id: datasetId, - }, - raw: raw - }); - return dataSource -} - -export const getDatasource = async (datasetId: string) => { - const dataSource = await Datasource.findOne({ - where: { - dataset_id: datasetId, - }, - }); - return dataSource -} - -export const getUpdatedSchema = async (configs: Record) => { - const { id, transformation_config, denorm_config, data_schema, action, indexCol = ingestionConfig.indexCol["Event Arrival Time"] } = configs - const existingTransformations = await DatasetTransformationsDraft.findAll({ where: { dataset_id: id }, raw: true }) - let resultantTransformations: any[] = [] - if (action === "edit") { - const toDeleteTransformations = _.compact(_.map(transformation_config, config => { - if (_.includes(["update", "remove"], _.get(config, "action"))) { - return _.get(config, ["value", "field_key"]) - } - })) - const updatedExistingTransformations = _.compact(_.map(existingTransformations, configs => { - if (!_.includes(toDeleteTransformations, _.get(configs, "field_key"))) { - return configs - } - })) || [] - const newTransformations = _.compact(_.map(transformation_config, config => { - if (_.includes(["update", "add"], _.get(config, "action"))) { - return config - } - })) || [] - resultantTransformations = [...updatedExistingTransformations, ...newTransformations] - } - if (action === "create") { - resultantTransformations = transformation_config || [] - } - let denormFields = _.get(denorm_config, "denorm_fields") - let updatedColumns = flattenSchema(data_schema) - const transformedFields = _.filter(resultantTransformations, field => _.get(field, ["metadata", "section"]) === "transformation") - let additionalFields = _.filter(resultantTransformations, field => _.get(field, ["metadata", "section"]) === "additionalFields") - updatedColumns = _.map(updatedColumns, (item) => { - const transformedData = _.find(transformedFields, { field_key: item.column }); - if (transformedData) { - const data = _.get(transformedData, "metadata") - return { - ...item, - type: _.get(data, "_transformedFieldSchemaType") || "string", - isModified: true, - ...data - }; - } - return item; - }); - denormFields = _.size(denormFields) ? await formatDenormFields(denormFields) : [] - additionalFields = formatNewFields(additionalFields, null); - let ingestionPayload = { schema: [...updatedColumns, ...denormFields, ...additionalFields] }; - if (indexCol === ingestionConfig.indexCol["Event Arrival Time"]) - ingestionPayload = { schema: [...updatedColumns, ...defaultTsObject, ...denormFields, ...additionalFields] }; - const updatedIngestionPayload = updateJSONSchema(data_schema, ingestionPayload) - return updatedIngestionPayload -} - -export const updateJSONSchema = (schema: Record, updatePayload: Record) => { - const clonedOriginal = _.cloneDeep(schema); - const modifiedRows = _.filter(_.get(updatePayload, "schema"), ["isModified", true]); - _.forEach(modifiedRows, modifiedRow => { - const { isDeleted = false, required = false, key, type, description = null, arrival_format, data_type, isModified = false } = modifiedRow; - if (isDeleted) { - deleteItemFromSchema(clonedOriginal, `${key}`, false); - } else { - updateTypeInSchema(clonedOriginal, `${key}`, type, true); - updateFormatInSchema(clonedOriginal, `${key}`, arrival_format); - updateDataTypeInSchema(clonedOriginal, `${key}`, data_type, isModified); - descriptionInSchema(clonedOriginal, `${key}`, description); - changeRequiredPropertyInSchema(clonedOriginal, `${key}`, required); - } - }); - return clonedOriginal; -} - -const updateDataTypeInSchema = (schema: Record, schemaPath: string, data_type: string, isModified: boolean) => { - const existing = _.get(schema, schemaPath); - if (isModified) { - const validDateFormats = ["date-time", "date", "epoch"] - if (!_.includes(validDateFormats, data_type)) { - _.unset(existing, "format"); - } else { - data_type === "epoch" ? _.set(existing, "format", "date-time") : _.set(existing, "format", data_type) - } - } - _.set(schema, schemaPath, { ...existing, data_type }); -} - - -const descriptionInSchema = (schema: Record, schemaPath: string, description: string) => { - const existing = _.get(schema, schemaPath); - if (description) _.set(schema, schemaPath, { ...existing, description }); -} - -const updateFormatInSchema = (schema: Record, schemaPath: string, arrival_format: string) => { - const existing = _.get(schema, schemaPath); - _.set(schema, schemaPath, { ...existing, arrival_format }); -} - -const updateTypeInSchema = (schema: Record, schemaPath: string, type: string, removeSuggestions: boolean = false) => { - const existing = _.get(schema, schemaPath); - if (removeSuggestions) { - _.unset(existing, "suggestions"); - _.unset(existing, "oneof"); - _.unset(existing, "arrivalOneOf") - } - _.set(schema, schemaPath, { ...existing, type }); -} - - -const deleteItemFromSchema = (schema: Record, schemaKeyPath: string, required: boolean) => { - if (_.has(schema, schemaKeyPath)) { - _.unset(schema, schemaKeyPath); - changeRequiredPropertyInSchema(schema, schemaKeyPath, required); - } -} - -const getPathToRequiredKey = (schema: Record, schemaKeyPath: string, schemaKey: string) => { - const regExStr = `properties.${schemaKey}`; - const regex = `(.${regExStr})`; - const [pathToRequiredKey] = _.split(schemaKeyPath, new RegExp(regex, "g")); - if (pathToRequiredKey === schemaKeyPath) return "required" - return `${pathToRequiredKey}.required` -} - -const changeRequiredPropertyInSchema = (schema: Record, schemaKeyPath: string, required: boolean) => { - const schemaKey = _.last(_.split(schemaKeyPath, ".")); - if (schemaKey) { - const pathToRequiredProperty = getPathToRequiredKey(schema, schemaKeyPath, schemaKey); - const existingRequiredKeys = _.get(schema, pathToRequiredProperty) || []; - if (required) { - // add to required property. - const updatedRequiredKeys = _.includes(existingRequiredKeys, schemaKey) ? existingRequiredKeys : [...existingRequiredKeys, schemaKey]; - _.set(schema, pathToRequiredProperty, updatedRequiredKeys); - } else { - // remove from required property. - const updatedRequiredKeys = _.difference(existingRequiredKeys, [schemaKey]); - if (_.size(updatedRequiredKeys) > 0) - _.set(schema, pathToRequiredProperty, updatedRequiredKeys); - } - } -} -export const formatNewFields = (newFields: Record, dataMappings: any) => { - if (newFields.length > 0) { - const final = _.map(newFields, (item: any) => { - const columnKey = _.join(_.map(_.split(_.get(item, "field_key"), "."), payload => `properties.${payload}`), ".") - return { - "column": item.field_key, - "type": _.get(item, ["metadata", "_transformedFieldSchemaType"]) || "string", - "key": columnKey, - "ref": columnKey, - "isModified": true, - "required": false, - "data_type": _.get(item, ["metadata", "_transformedFieldDataType"]), - ...(dataMappings && { "arrival_format": getArrivalFormat(_.get(item, "_transformedFieldSchemaType"), dataMappings) || _.get(item, "arrival_format") }) - } - }); - return final; - } - else return []; -} -const getArrivalFormat = (data_type: string | undefined, dataMappings: Record) => { - let result = null; - if (data_type) { - _.forEach(dataMappings, (value, key) => { - if (_.includes(_.get(value, "arrival_format"), data_type)) { - result = key; - } - }); - } - return result; -} -export const updateDenormDerived = (schemaColumns: any, columns: any, fixedPrefix: string): any[] => { - const result = _.map(columns, (column: any) => { - const isExistingColumn = _.find(schemaColumns, ["column", column.field_key]); - if (isExistingColumn) { - return { - ...isExistingColumn, - "type": _.get(column, "metadata._transformedFieldSchemaType"), - "data_type": _.get(column, "metadata._transformedFieldDataType"), - "required": false, - "isModified": true, - ..._.get(column, "metadata"), - }; - } else { - const columnKey = _.join(_.map(_.split(_.get(column, "field_key"), "."), payload => `properties.${payload}`), ".") - return { - "column": `${fixedPrefix}.${column.field_key}`, - "type": _.get(column, "metadata._transformedFieldSchemaType"), - "key": `properties.${fixedPrefix}.${columnKey}`, - "ref": `properties.${fixedPrefix}.${columnKey}`, - "required": false, - "isModified": true, - "data_type": _.get(column, "metadata._transformedFieldDataType"), - ..._.get(column, "metadata"), - }; - } - }); - return _.concat(schemaColumns, result); -} -const processDenormConfigurations = async (item: any) => { - const denormFieldsData: any = []; - const redis_db = _.get(item, "redis_db"); - const denorm_out_field = _.get(item, "denorm_out_field"); - const dataset: any = await Dataset.findOne({ where: { "dataset_config": { "redis_db": redis_db } }, raw: true }) || [] - const transformations = _.size(dataset) ? await DatasetTransformations.findAll({ where: { status: DatasetStatus.Live, dataset_id: _.get(dataset, "dataset_id") }, raw: true }) : [] - let schema = flattenSchema(_.get(dataset, "data_schema"), denorm_out_field, true); - schema = updateDenormDerived(schema, _.get(transformations, "data.result"), denorm_out_field,); - denormFieldsData.push({ - "column": denorm_out_field, - "type": "object", - "key": `properties.${denorm_out_field}`, - "ref": `properties.${denorm_out_field}`, - "isModified": true, - "required": false, - "arrival_format": "object", - "data_type": "object", - }); - denormFieldsData.push(...schema); - return denormFieldsData; -} -export const formatDenormFields = async (denormFields: any) => { - if (denormFields.length > 0) { - const final = _.map(denormFields, (item: any) => { - return processDenormConfigurations(item); - }); - return Promise.all(final).then((data) => _.flatten(data)); - } - else return []; -} -const addRequiredFields = ( - type: string, - result: Record, - schemaObject: Record, - required: string[], -) => { - const requiredFields = _.get(schemaObject, "required") || []; - _.map(result, (item) => { - if (type === "array" || type === "object") { - if (required && required.includes(item.key.replace("properties.", ""))) item.required = true; - else if (requiredFields.includes(item.key.replace("properties.", ""))) item.required = true; - else item.required = false; - } - else if (requiredFields.includes(item.key.replace("properties.", ""))) item.required = true; - else item.required = false; - }) -} -const flatten = (schemaObject: Record, rollup: boolean = false) => { - const schemaObjectData = schemaObject; - const result: Record = {}; - const getKeyName = (prefix: string, key: string) => prefix ? `${prefix}.${key}` : key; - const flattenHelperFn = (propertySchema: Record, prefix: string, ref: string, arrayChild = false) => { - const { type, properties, items, required = false, ...rest } = propertySchema || {}; - if (type === "object" && properties) { - if (prefix !== "" && !arrayChild) result[prefix] = { type, key: ref, ref, properties, items, parent: true, ...rest }; - for (const [key, value] of Object.entries(properties)) { - flattenHelperFn(value as Record, getKeyName(prefix, key), getKeyName(ref, `properties.${key}`)); - } - } else if (type === "array" && items && !rollup) { - if (prefix !== "") result[prefix] = { type, key: ref, ref, properties, items, parent: true, ...rest }; - if (["array", "object"].includes(items?.type)) { - flattenHelperFn(items, prefix, getKeyName(ref, `items`), true) - } else { - result[prefix] = { type, key: ref, ref, properties, items, ...rest }; - } - } else { - result[prefix] = { type, key: ref, ref, properties, items, ...rest }; - } - addRequiredFields(type, result, schemaObjectData, required); - } - flattenHelperFn(schemaObjectData, "", ""); - return result; -} -export const flattenSchema = (schema: Record, fixedPrefix?: string | undefined, modified?: boolean, rollup: boolean = false) => { - const flattend = flatten(schema, rollup); - if (fixedPrefix) - return _.map(flattend, (value, key) => { - const { key: propertyKey, ref } = value; - const keySplit = _.split(propertyKey, "."); - const refSplit = _.split(ref, "."); - keySplit.splice(1, 0, fixedPrefix, "properties"); - refSplit.splice(1, 0, fixedPrefix, "properties"); - const data = { - column: `${fixedPrefix}.${key}`, - ...value, - key: keySplit.join("."), - ref: refSplit.join("."), - }; - if (modified) { data.isModified = true; data.required = false; } - return data; - }); - return _.map(flattend, (value, key) => ({ column: key, ...value })); -} \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index 6191d43e..cfd333eb 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -101,6 +101,9 @@ describe("DATASET READ API", () => { chai.spy.on(ConnectorInstances, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.MASTER_DATASET_SCHEMA) + }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) }) @@ -133,6 +136,9 @@ describe("DATASET READ API", () => { chai.spy.on(DatasetSourceConfig, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.MASTER_DATASET_SCHEMA) + }) chai.spy.on(DatasetDraft, "create", () => { return Promise.resolve({ dataValues: TestInputsForDatasetRead.DRAFT_SCHEMA }) }) @@ -217,6 +223,66 @@ describe("DATASET READ API", () => { }); }); + it("Dataset read failure: When dependent denorm master dataset not found", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "api_version": "v1" }) + }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("The dependent dataset not found") + res.body.error.code.should.be.eq("DEPENDENT_MASTER_DATA_NOT_FOUND") + done(); + }); + }); + + it("Dataset read failure: When dependent denorm master dataset not live", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve() + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ ...TestInputsForDatasetRead.LIVE_SCHEMA, "api_version": "v1" }) + }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA_V1) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{"dataset_id":"master_dataset", "dataset_config":{"cache_config":{"redis_db":20}}}]) + }) + chai + .request(app) + .get("/v2/datasets/read/sb-telemetry?mode=edit") + .end((err, res) => { + res.should.have.status(httpStatus.PRECONDITION_REQUIRED); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.error.message.should.be.eq("The dependent master dataset is not published") + res.body.error.code.should.be.eq("DEPENDENT_MASTER_DATA_NOT_LIVE") + done(); + }); + }); + it("Dataset read failure: When the dataset of requested dataset_id not found", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve(null) diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts index 2efaec73..d520a102 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/Fixtures.ts @@ -78,6 +78,15 @@ export const TestInputsForDatasetRead = { ], "data_version": 1, "api_version": "v2", + "denorm_config": { + "denorm_fields": [ + { + "denorm_key": "actor.id", + "denorm_out_field": "userdata", + "redis_db": 16 + } + ] + }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, @@ -92,6 +101,7 @@ export const TestInputsForDatasetRead = { ] } }, + MASTER_DATASET_SCHEMA:[{"dataset_id":"master_dataset", "dataset_config":{"cache_config":{"redis_db":16}}}], TRANSFORMATIONS_SCHEMA: [{ "field_key": "eid", "transformation_function": { "type": "mask", "expr": "eid", "datatype": "string", "category": "pii" }, "mode": "Strict" }], TRANSFORMATIONS_SCHEMA_V1: [ { diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index cd32573f..59598719 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -66,6 +66,28 @@ describe("DATASET STATUS TRANSITION LIVE", () => { }); }); + it("Dataset status transition failure: Unable to fetch redis db number for master dataset", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_INVALID) + }) + chai.spy.on(sequelize, "query", () => { + return Promise.resolve([]) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) + .end((err, res) => { + res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Unable to fetch the redis db index for the master data") + res.body.error.code.should.be.eq("REDIS_DB_INDEX_FETCH_FAILED") + done(); + }); + }); it("Dataset status transition success: When the action is to set master dataset live", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { @@ -152,29 +174,6 @@ describe("DATASET STATUS TRANSITION LIVE", () => { }); }); - it("Dataset status transition failure: Unable to fetch redis db number for master dataset", (done) => { - chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_INVALID) - }) - chai.spy.on(sequelize, "query", () => { - return Promise.resolve([]) - }) - chai - .request(app) - .post("/v2/datasets/status-transition") - .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE_MASTER) - .end((err, res) => { - res.should.have.status(httpStatus.INTERNAL_SERVER_ERROR); - res.body.should.be.a("object") - res.body.id.should.be.eq("api.datasets.status-transition"); - res.body.params.status.should.be.eq("FAILED") - res.body.params.msgid.should.be.eq(msgid) - res.body.error.message.should.be.eq("Unable to fetch the redis db index for the master data") - res.body.error.code.should.be.eq("REDIS_DB_INDEX_FETCH_FAILED") - done(); - }); - }); - it("Dataset status transition failure: When dataset is not found to publish", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve() From f07caf5c5ccd02ea0c84c244c06c27cc16d896a8 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 21 Aug 2024 11:46:55 +0530 Subject: [PATCH 149/235] #OBS-I146: fix: Hudi spec generation test cases --- api-service/src/services/DatasourceService.ts | 1 - .../DatasetLive.spec.ts | 88 +++++++++++++++++++ .../DatasetStatusTransition/Fixtures.ts | 1 + 3 files changed, 89 insertions(+), 1 deletion(-) diff --git a/api-service/src/services/DatasourceService.ts b/api-service/src/services/DatasourceService.ts index f9f6b2a7..812a5611 100644 --- a/api-service/src/services/DatasourceService.ts +++ b/api-service/src/services/DatasourceService.ts @@ -1,4 +1,3 @@ -import _ from "lodash"; import { Datasource } from "../models/Datasource"; export const getDatasourceList = async (datasetId: string, raw = false) => { diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 59598719..7c787387 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -11,6 +11,7 @@ import { commandHttpService } from "../../../connections/commandServiceConnectio import { sequelize } from "../../../connections/databaseConnection"; import { DatasourceDraft } from "../../../models/DatasourceDraft"; import { Dataset } from "../../../models/Dataset"; +import { Datasource } from "../../../models/Datasource"; chai.use(spies); chai.should(); @@ -66,6 +67,93 @@ describe("DATASET STATUS TRANSITION LIVE", () => { }); }); + it("Dataset status transition success: When the action is to set dataset live v1 by creating hudi spec", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH_HUDI) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + + it("Dataset status transition success: When the action is to set dataset live v2 by updating hudi spec", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_DATASET_SCHEMA_FOR_PUBLISH_HUDI) + }) + chai.spy.on(Dataset, "findAll", () => { + return Promise.resolve([{ "id": "master-dataset", "status": "Live", "dataset_config": { "cache_config": { "redis_db": 21 } }, "api_version": "v2" }]) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({}) + }) + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ "api_version":"v2", "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) + }) + chai.spy.on(DatasourceDraft, "create", () => { + return Promise.resolve({}) + }) + chai.spy.on(Datasource, "findOne", () => { + return Promise.resolve({"ingestion_spec":{"dataset": "dataset-all-fields4", "schema": {"table": "dataset-all-fields4_events", "partitionColumn": "eid", "timestampColumn": "obsrv_meta.syncts", "primaryKey": "eid", "columnSpec": [{"type": "string", "name": "mid", "index": 1}, {"type": "epoch", "name": "ets", "index": 2}, {"type": "string", "name": "userdata.mid", "index": 3}, {"type": "epoch", "name": "userdata.ets", "index": 4}, {"type": "string", "name": "userdata.eid", "index": 5}, {"type": "string", "name": "email", "index": 6}, {"type": "string", "name": "obsrv.meta.source.connector", "index": 7}, {"type": "string", "name": "obsrv.meta.source.id", "index": 8}]}, "inputFormat": {"type": "json", "flattenSpec": {"fields": [{"type": "path", "expr": "$.mid", "name": "mid"}, {"type": "path", "expr": "$.ets", "name": "ets"}, {"type": "path", "expr": "$.eid", "name": "eid"}, {"type": "path", "expr": "$.userdata.mid", "name": "userdata.mid"}, {"type": "path", "expr": "$.userdata.ets", "name": "userdata.ets"}, {"type": "path", "expr": "$.userdata.eid", "name": "userdata.eid"}, {"type": "path", "expr": "$.email", "name": "email"}, {"type": "path", "expr": "$.obsrv_meta.syncts", "name": "obsrv_meta.syncts"}, {"type": "path", "expr": "$.obsrv_meta.source.connector", "name": "obsrv.meta.source.connector"}, {"type": "path", "expr": "$.obsrv_meta.source.connectorInstance", "name": "obsrv.meta.source.id"}]}}}}) + }) + const t = chai.spy.on(sequelize, "transaction", () => { + return Promise.resolve(sequelize.transaction) + }) + chai.spy.on(t, "commit", () => { + return Promise.resolve({}) + }) + chai.spy.on(commandHttpService, "post", () => { + return Promise.resolve({}) + }) + chai + .request(app) + .post("/v2/datasets/status-transition") + .send(TestInputsForDatasetStatusTransition.VALID_SCHEMA_FOR_LIVE) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq("api.datasets.status-transition"); + res.body.params.status.should.be.eq("SUCCESS") + res.body.result.should.be.a("object") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.message.should.be.eq("Dataset status transition to Live successful") + res.body.result.dataset_id.should.be.eq("telemetry") + done(); + }); + }); + it("Dataset status transition failure: Unable to fetch redis db number for master dataset", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(TestInputsForDatasetStatusTransition.DRAFT_MASTER_DATASET_INVALID) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts index ecf531f0..e65f1852 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/Fixtures.ts @@ -158,6 +158,7 @@ export const TestInputsForDatasetStatusTransition = { "api_version": "v2" }, DRAFT_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": true, "lakehouse_enabled": false, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, + DRAFT_DATASET_SCHEMA_FOR_PUBLISH_HUDI: { "dataset_id": "telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "telemetry", "type": "events", "api_version": "v2", "denorm_config": { "denorm_fields": [{ "denorm_out_field": "pid", "denorm_key": "eid", "dataset_id": "master-dataset" }] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": true, "cache_enabled": false }, "keys_config": { "timestamp_key": "ets", "partition_key": "ets", "data_key": "eid" }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, DRAFT_MASTER_DATASET_SCHEMA_FOR_PUBLISH: { "dataset_id": "master-telemetry", "data_schema": { "$schema": "https://json-schema.org/draft/2020-12/schema", "type": "object", "properties": { "ets": { "type": "string" }, "ver": { "type": "string" } }, "additionalProperties": true }, "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } }, DRAFT_MASTER_DATASET_INVALID: { "dataset_id": "master-telemetry", "status": "ReadyToPublish", "id": "master-telemetry", "type": "master", "api_version": "v2", "denorm_config": { "denorm_fields": [] }, "dataset_config": { "indexing_config": { "olap_store_enabled": false, "lakehouse_enabled": false, "cache_enabled": true }, "keys_config": { "timestamp_key": "ets", "partition_key": "", "data_key": "eid" }, "cache_config": { "redis_db_host": "localhost", "redis_db_port": 5679, "redis_db": 0 }, "file_upload_path": ["telemetry.json"] }, "router_config": { "topic": "telemetry" } } } \ No newline at end of file From 802eb731c76ad41620d876c54545b29c823d4537 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 21 Aug 2024 23:31:09 +0530 Subject: [PATCH 150/235] #OBS-I146: fix: Test case and linting fix --- api-service/.eslintrc | 3 +- api-service/src/middlewares/errors.ts | 4 +- .../DatasetLive.spec.ts | 6 -- .../DatasetRetire.spec.ts | 6 -- .../DatasetUpdate/DatasetDenorm.spec.ts | 32 ++++++++++ .../DatasetUpdate/DatasetUpdate.spec.ts | 62 +++++++++---------- .../DatasetUpdate/Fixtures.ts | 23 ++++++- 7 files changed, 89 insertions(+), 47 deletions(-) diff --git a/api-service/.eslintrc b/api-service/.eslintrc index 598451ab..b209a5f4 100644 --- a/api-service/.eslintrc +++ b/api-service/.eslintrc @@ -9,7 +9,8 @@ "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended" ], - "rules": { + "rules": { + "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], "@typescript-eslint/no-explicit-any": ["off"], "@typescript-eslint/no-useless-escape": ["off"], "@typescript-eslint/quotes": [ diff --git a/api-service/src/middlewares/errors.ts b/api-service/src/middlewares/errors.ts index 96c8069e..ce3200ab 100644 --- a/api-service/src/middlewares/errors.ts +++ b/api-service/src/middlewares/errors.ts @@ -4,7 +4,7 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; import _ from "lodash"; import { ObsrvError } from "../types/ObsrvError"; -export const errorHandler = (err: Error, req: Request, res: Response, next: NextFunction) => { +export const errorHandler = (err: Error, req: Request, res: Response, _next: NextFunction) => { logger.error({ path: req.url, req: req.body , ...err }) const errorMessage = {name: err.name, message: err.message}; @@ -12,7 +12,7 @@ export const errorHandler = (err: Error, req: Request, res: Response, next: Next }; -export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, next: NextFunction) => { +export const obsrvErrorHandler = (obsrvErr: ObsrvError, req: Request, res: Response, _next: NextFunction) => { logger.error({ path: req.url, req: req.body, resmsgid: _.get(res, "resmsgid") , ...obsrvErr }) ResponseHandler.obsrvErrorResponse(obsrvErr, req, res); diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index 7c787387..e26ddafe 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -187,12 +187,6 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({}) }) - chai.spy.on(Dataset, "findOne", () => { - return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) - }) - chai.spy.on(DatasourceDraft, "create", () => { - return Promise.resolve({}) - }) const t = chai.spy.on(sequelize, "transaction", () => { return Promise.resolve(sequelize.transaction) }) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index 0be1fdb9..1a0428af 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -91,12 +91,6 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Dataset, "update", () => { return Promise.resolve({}) }) - chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve() - }) - chai.spy.on(DatasetDraft, "findAll", () => { - return Promise.resolve() - }) chai.spy.on(commandHttpService, "post", () => { return Promise.resolve({}) }) diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts index afc673ca..5f330032 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetDenorm.spec.ts @@ -103,4 +103,36 @@ describe("DATASET DENORM UPDATE", () => { }); }); + it("Success: Ignore the denorm when payload contains denorm that already exists", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ + id: "telemetry", version_key: validVersionKey, type:"dataset", api_version:"v2", status: "Draft", denorm_config: { + denorm_fields: [ { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_id": "master" + }] + } + }) + }) + chai.spy.on(DatasetDraft, "update", () => { + return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) + }) + chai + .request(app) + .patch("/v2/datasets/update") + .send(TestInputsForDatasetUpdate.DATASET_UPDATE_WITH_EXISTING_DENORM) + .end((err, res) => { + res.should.have.status(httpStatus.OK); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("SUCCESS") + res.body.params.msgid.should.be.eq(msgid) + res.body.result.id.should.be.eq("telemetry") + res.body.result.message.should.be.eq("Dataset is updated successfully") + res.body.result.version_key.should.be.a("string") + done(); + }); + }); + }) \ No newline at end of file diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts index 6cb84e25..c2050f23 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/DatasetUpdate.spec.ts @@ -29,7 +29,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -61,7 +61,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -80,7 +80,7 @@ describe("DATASET UPDATE API", () => { }); it("Dataset updation failure: When no fields with dataset_id is provided in the request payload", (done) => { - + chai .request(app) .patch("/v2/datasets/update") @@ -101,7 +101,7 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "findOne", () => { return Promise.resolve(null) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -120,9 +120,9 @@ describe("DATASET UPDATE API", () => { it("Dataset updation failure: When dataset to update is outdated", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", type:"event", status: "Draft", version_key: "1813444815918", api_version: "v2" }) + return Promise.resolve({ id: "telemetry", type: "event", status: "Draft", version_key: "1813444815918", api_version: "v2" }) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -139,11 +139,32 @@ describe("DATASET UPDATE API", () => { }); }); + it("Dataset updation failure: When dataset to update is of api_version v1", (done) => { + chai.spy.on(DatasetDraft, "findOne", () => { + return Promise.resolve({ id: "telemetry", type: "event", status: "Draft", version_key: "1813444815918", api_version: "v1" }) + }) + + chai + .request(app) + .patch("/v2/datasets/update") + .send({ ...requestStructure, request: { dataset_id: "telemetry", version_key: validVersionKey, name: "telemetry" } }) + .end((err, res) => { + res.should.have.status(httpStatus.NOT_FOUND); + res.body.should.be.a("object") + res.body.id.should.be.eq(apiId); + res.body.params.status.should.be.eq("FAILED") + res.body.params.msgid.should.be.eq(msgid) + res.body.error.message.should.be.eq("Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset") + res.body.error.code.should.be.eq("DATASET_API_VERSION_MISMATCH") + done(); + }); + }); + it("Dataset updation failure: Dataset to update is not in draft state", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", type:"event", status: "Live", version_key: "1713444815918", api_version: "v2" }) + return Promise.resolve({ id: "telemetry", type: "event", status: "Live", version_key: "1713444815918", api_version: "v2" }) }) - + chai .request(app) .patch("/v2/datasets/update") @@ -187,17 +208,11 @@ describe("DATASET UPDATE API", () => { it("Success: Dataset name updated successfully", (done) => { chai.spy.on(DatasetDraft, "findOne", () => { - return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) + return Promise.resolve({ id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -216,7 +231,7 @@ describe("DATASET UPDATE API", () => { }); it("Failure: Failed to update the dataset name", (done) => { - + chai .request(app) .patch("/v2/datasets/update") @@ -246,18 +261,9 @@ describe("DATASET UPDATE API", () => { id: "telemetry", status: "Draft", version_key: validVersionKey, type: "event", api_version: "v2" }) }) - chai.spy.on(DatasetTransformationsDraft, "findAll", () => { - return Promise.resolve([{ field_key: "key2" }, { field_key: "key3" }]) - }) chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") @@ -308,12 +314,6 @@ describe("DATASET UPDATE API", () => { chai.spy.on(DatasetDraft, "update", () => { return Promise.resolve({ dataValues: { id: "telemetry", message: "Dataset is updated successfully" } }) }) - const t = chai.spy.on(sequelize, "transaction", () => { - return Promise.resolve(sequelize.transaction) - }) - chai.spy.on(t, "commit", () => { - return Promise.resolve({}) - }) chai .request(app) .patch("/v2/datasets/update") diff --git a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts index 97248d9f..ae9628a4 100644 --- a/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts +++ b/api-service/src/tests/DatasetManagement/DatasetUpdate/Fixtures.ts @@ -15,7 +15,8 @@ export const TestInputsForDatasetUpdate = { ...requestStructure, request: { "dataset_id": "telemetry", "version_key": validVersionKey, - "name": "telemetry" + "name": "telemetry", + "sample_data":{"events":{}} } }, @@ -383,6 +384,26 @@ export const TestInputsForDatasetUpdate = { } }, + DATASET_UPDATE_WITH_EXISTING_DENORM: { + ...requestStructure, request: { + "dataset_id": "telemetry", + "version_key": validVersionKey, + "name": "sb-telemetry", + "denorm_config": { + "denorm_fields": [ + { + "value": { + "denorm_key": "actor.id", + "denorm_out_field": "mid", + "dataset_id": "master" + }, + "action": "upsert" + } + ] + } + } + }, + DATASET_UPDATE_WITH_SAME_TRANSFORMATION_ADD_REMOVE: { ...requestStructure, request: { "dataset_id": "telemetry", From 2b4740c077e01bf9370b7781a176c7c3615de68f Mon Sep 17 00:00:00 2001 From: yashashk Date: Thu, 22 Aug 2024 15:57:11 +0530 Subject: [PATCH 151/235] merge commit --- .../DatasetCopy/DatasetCopyHelper.ts | 1 - .../RequestValidationSchemaV2.json | 3 + api-service/src/services/DatasetService.ts | 100 ++++++++++-------- 3 files changed, 56 insertions(+), 48 deletions(-) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts index de4c8087..82b35ebd 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopyHelper.ts @@ -10,7 +10,6 @@ export const updateRecords = (datasetRecord: Record, newDatasetId: _.set(datasetRecord, 'status', DatasetStatus.Draft) _.set(datasetRecord, "dataset_id", dataset_id) _.set(datasetRecord, "id", dataset_id) - _.set(datasetRecord, "name", dataset_id) _.set(datasetRecord, "version_key", Date.now().toString()) _.set(datasetRecord, 'version', version); _.set(datasetRecord, "entry_topic", config.telemetry_service_config.kafka.topics.createDataset) diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json index 45820502..da61770c 100644 --- a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json @@ -77,6 +77,9 @@ "type": "boolean", "default": true }, + "batch_id": { + "type": "string" + }, "extraction_key": { "type": "string", "default": "events" diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 25b120f2..0dfee5b6 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -25,7 +25,7 @@ class DatasetService { } findDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Dataset.findAll({where, attributes, order, raw: true}) + return Dataset.findAll({ where, attributes, order, raw: true }) } getDuplicateDenormKey = (denormConfig: Record): Array => { @@ -39,9 +39,9 @@ class DatasetService { } checkDatasetExists = async (dataset_id: string): Promise => { - const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes:["id"], raw: true }); + const draft = await DatasetDraft.findOne({ where: { dataset_id }, attributes: ["id"], raw: true }); if (draft === null) { - const live = await Dataset.findOne({ where: { id: dataset_id }, attributes:["id"], raw: true }); + const live = await Dataset.findOne({ where: { id: dataset_id }, attributes: ["id"], raw: true }); return !(live === null) } else { return true; @@ -53,7 +53,7 @@ class DatasetService { } findDraftDatasets = async (where?: Record, attributes?: string[], order?: any): Promise => { - return DatasetDraft.findAll({where, attributes, order, raw: true}) + return DatasetDraft.findAll({ where, attributes, order, raw: true }) } getDraftTransformations = async (dataset_id: string, attributes?: string[]) => { @@ -68,7 +68,7 @@ class DatasetService { return DatasetSourceConfig.findAll({ where: { dataset_id }, attributes, raw: true }); } - getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { + getConnectors = async (dataset_id: string, attributes?: string[]): Promise> => { return ConnectorInstances.findAll({ where: { dataset_id }, attributes, raw: true }); } @@ -78,7 +78,7 @@ class DatasetService { updateDraftDataset = async (draftDataset: Record): Promise> => { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }}); + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } }); const responseData = { message: "Dataset is updated successfully", id: draftDataset.id, version_key: draftDataset.version_key }; logger.info({ draftDataset, message: `Dataset updated successfully with id:${draftDataset.id}`, response: responseData }); return responseData; @@ -96,7 +96,6 @@ class DatasetService { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); const transaction = await sequelize.transaction(); - try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); await DatasetTransformationsDraft.destroy({ where: { dataset_id }, transaction }); @@ -125,12 +124,14 @@ class DatasetService { const transformationFields = ["field_key", "transformation_function", "mode", "metadata"] const transformations = _.includes([DatasetStatus.Live], status) ? await this.getTransformations(dataset_id, transformationFields) : await this.getDraftTransformations(dataset_id, transformationFields); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, ["field_key"]), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, ["metadata._transformedFieldDataType"]) || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section) }, mode: _.get(config, ["mode"]) } @@ -147,31 +148,34 @@ class DatasetService { }) draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) draftDataset["sample_data"] = dataset_config?.mergedEvent + draftDataset["status"] = DatasetStatus.Draft return draftDataset; } - getTransformationCategory = (section: string):string => { + getTransformationCategory = (section: string): string => { - switch(section) { + switch (section) { case "pii": return "pii"; case "additionalFields": return "derived"; + case "derived": + return "derived"; default: return "transform"; } } createDraftDatasetFromLive = async (dataset: Model) => { - - let draftDataset:any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); - const dataset_config:any = _.get(dataset, "dataset_config"); - const api_version:any = _.get(dataset, "api_version"); - if(api_version === "v1") { + + let draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); + const dataset_config: any = _.get(dataset, "dataset_config"); + const api_version: any = _.get(dataset, "api_version"); + if (api_version === "v1") { draftDataset["dataset_config"] = { - indexing_config: {olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master")}, - keys_config: {data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key}, - cache_config: {redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db} + indexing_config: { olap_store_enabled: true, lakehouse_enabled: false, cache_enabled: (_.get(dataset, "type") === "master") }, + keys_config: { data_key: dataset_config.data_key, timestamp_key: dataset_config.timestamp_key }, + cache_config: { redis_db_host: dataset_config.redis_db_host, redis_db_port: dataset_config.redis_db_port, redis_db: dataset_config.redis_db } } const connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); draftDataset["connectors_config"] = _.map(connectors, (config) => { @@ -184,12 +188,14 @@ class DatasetService { }) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode", "metadata"]); draftDataset["transformations_config"] = _.map(transformations, (config) => { + const section: any = _.get(config, "metadata.section"); + config = _.omit(config, "transformation_function.condition") return { field_key: _.get(config, "field_key"), transformation_function: { ..._.get(config, ["transformation_function"]), datatype: _.get(config, "metadata._transformedFieldDataType") || "string", - category: this.getTransformationCategory(_.get(config, ["metadata.section"])) + category: this.getTransformationCategory(section), }, mode: _.get(config, "mode") } @@ -205,14 +211,14 @@ class DatasetService { } const denormConfig = _.get(draftDataset, "denorm_config") if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { - const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id","dataset_id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live, type: "master" }, ["id", "dataset_id", "status", "dataset_config", "api_version"]) if (_.isEmpty(masterDatasets)) { throw { code: "DEPENDENT_MASTER_DATA_NOT_FOUND", message: `The dependent dataset not found`, errCode: "NOT_FOUND", statusCode: 404 } } const updatedDenormFields = _.map(denormConfig.denorm_fields, field => { const { redis_db, denorm_out_field, denorm_key } = field let masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.cache_config.redis_db") === redis_db) - if(!masterConfig){ + if (!masterConfig) { masterConfig = _.find(masterDatasets, data => _.get(data, "dataset_config.redis_db") === redis_db) } if (_.isEmpty(masterConfig)) { @@ -238,12 +244,12 @@ class DatasetService { const { id } = dataset const transaction = await sequelize.transaction() try { - await DatasetTransformationsDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasourceDraft.destroy({ where: { dataset_id: id } , transaction}) - await DatasetDraft.destroy({ where: { id } , transaction}) + await DatasetTransformationsDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetSourceConfigDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasourceDraft.destroy({ where: { dataset_id: id }, transaction }) + await DatasetDraft.destroy({ where: { id }, transaction }) await transaction.commit() - } catch (err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(dataset.id, "FAILED_TO_DELETE_DATASET", err.message, "SERVER_ERROR", 500, err) } @@ -255,18 +261,18 @@ class DatasetService { try { await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id } , transaction}); + await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); await this.deleteDruidSupervisors(dataset); - } catch(err:any) { + } catch (err: any) { await transaction.rollback(); throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); } } findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { - return Datasource.findAll({where, attributes, order, raw: true}) + return Datasource.findAll({ where, attributes, order, raw: true }) } private deleteDruidSupervisors = async (dataset: Record) => { @@ -290,26 +296,26 @@ class DatasetService { const indexingConfig = draftDataset.dataset_config.indexing_config; const transaction = await sequelize.transaction() try { - await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id } , transaction}) - if(indexingConfig.olap_store_enabled) { + await DatasetDraft.update(draftDataset, { where: { id: draftDataset.id }, transaction }) + if (indexingConfig.olap_store_enabled) { await this.createDruidDataSource(draftDataset, transaction); } - if(indexingConfig.lakehouse_enabled) { + if (indexingConfig.lakehouse_enabled) { const liveDataset = await this.getDataset(draftDataset.dataset_id, ["id", "api_version"], true); - if(liveDataset && liveDataset.api_version === "v2") { + if (liveDataset && liveDataset.api_version === "v2") { await this.updateHudiDataSource(draftDataset, transaction) } else { await this.createHudiDataSource(draftDataset, transaction) } } await transaction.commit() - } catch(err:any) { + } catch (err: any) { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } await executeCommand(draftDataset.id, "PUBLISH_DATASET"); - + } private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -318,7 +324,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -327,25 +333,25 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); - const dsId = _.join([draftDataset.dataset_id,"events","hudi"], "_") - const liveDatasource = await Datasource.findOne({where: {id: dsId}, attributes: ["ingestion_spec"], raw: true}) as unknown as Record + const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") + const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, 'ingestion_spec', ingestionSpec) - await DatasourceDraft.create(draftDatasource, {transaction}) + await DatasourceDraft.create(draftDatasource, { transaction }) } - private createDraftDatasource = (draftDataset: Record, type: string) : Record => { + private createDraftDatasource = (draftDataset: Record, type: string): Record => { - const datasource = _.join([draftDataset.dataset_id,"events"], "_") + const datasource = _.join([draftDataset.dataset_id, "events"], "_") return { - id: _.join([datasource,type], '_'), + id: _.join([datasource, type], '_'), datasource: draftDataset.dataset_id, dataset_id: draftDataset.dataset_id, datasource_ref: datasource, @@ -356,15 +362,15 @@ class DatasetService { } export const getLiveDatasetConfigs = async (dataset_id: string) => { - + let datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) - if(!_.isEmpty(transformations)){ + if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations } - if(!_.isEmpty(connectors)){ + if (!_.isEmpty(connectors)) { datasetRecord["connectors_config"] = connectors } return datasetRecord; From 82be980889edd23427abf92551cc6888ba267c40 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 22 Aug 2024 16:02:42 +0530 Subject: [PATCH 152/235] #OBS-I173: fix: Dataset web console required fixes --- .../RequestValidationSchemaV2.json | 3 +++ .../ReadyToPublishSchema.json | 6 ++---- api-service/src/services/DatasetService.ts | 17 ++++++++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json index 45820502..da61770c 100644 --- a/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json +++ b/api-service/src/controllers/DatasetImport/RequestValidationSchemaV2.json @@ -77,6 +77,9 @@ "type": "boolean", "default": true }, + "batch_id": { + "type": "string" + }, "extraction_key": { "type": "string", "default": "events" diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index f378a5f7..bbb67fdb 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -324,8 +324,7 @@ "minLength": 1 }, "expr": { - "type": "string", - "minLength": 1 + "type": "string" }, "condition": { "type": "object", @@ -335,8 +334,7 @@ "minLength": 1 }, "expr": { - "type": "string", - "minLength": 1 + "type": "string" } }, "required": ["type", "expr"], diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 28c1684e..71701e82 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -376,7 +376,9 @@ export const getLiveDatasetConfigs = async (dataset_id: string) => { let datasetRecord = await datasetService.getDataset(dataset_id, undefined, true) const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) - const connectors = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) + const connectorsV2 = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) + const connectorsV1 = await getV1Connectors(dataset_id) + const connectors = _.concat(connectorsV1,connectorsV2) if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations @@ -387,4 +389,17 @@ export const getLiveDatasetConfigs = async (dataset_id: string) => { return datasetRecord; } +export const getV1Connectors = async (datasetId: string) => { + const v1connectors = await datasetService.getConnectorsV1(datasetId, ["id", "connector_type", "connector_config"]); + const modifiedV1Connectors = _.map(v1connectors, (config) => { + return { + id: _.get(config, "id"), + connector_id: _.get(config, "connector_type"), + connector_config: _.get(config, "connector_config"), + version: "v1" + } + }) + return modifiedV1Connectors; +} + export const datasetService = new DatasetService(); \ No newline at end of file From 671a842dfd0f9630131a281d9eaae871f8f6d010 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Thu, 22 Aug 2024 18:31:58 +0530 Subject: [PATCH 153/235] #OBS-I173: fix: Dataset update changes to accept type changes --- api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts | 1 + .../DatasetUpdate/DatasetUpdateValidationSchema.json | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 25c9a633..590ea9b5 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -79,6 +79,7 @@ const mergeDraftDataset = (datasetModel: Model | null, datasetReq: any if(datasetReq.connectors_config) dataset["connectors_config"] = mergeConnectorsConfig(_.get(datasetModel, ["connectors_config"]), datasetReq.connectors_config) if(datasetReq.tags) dataset["tags"] = mergeTags(_.get(datasetModel, ["tags"]), datasetReq.tags) if(datasetReq.sample_data) dataset["sample_data"] = datasetReq.sample_data + if(datasetReq.type) dataset["type"] = datasetReq.type return dataset; } diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json index 4dbc3e89..b31fe674 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdateValidationSchema.json @@ -32,6 +32,10 @@ "version_key": { "type": "string" }, + "type": { + "type": "string", + "enum": ["event", "transaction", "master"] + }, "name": { "type": "string", "minLength": 1 @@ -199,7 +203,7 @@ } }, "additionalProperties": false - }, + }, "keys_config": { "type": "object", "properties": { From 35e47d565b86ab606d2e221fc14116db5355885d Mon Sep 17 00:00:00 2001 From: yashashk Date: Fri, 23 Aug 2024 10:47:09 +0530 Subject: [PATCH 154/235] #OBS-I167 : dataset read api changes to read live dataset source configs --- .../src/controllers/DatasetRead/DatasetRead.ts | 17 +++++------------ .../DatasetStatusTransition.ts | 2 +- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 2f6f3855..8bae2d9d 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -3,7 +3,7 @@ import httpStatus from "http-status"; import _ from "lodash"; import { ResponseHandler } from "../../helpers/ResponseHandler"; import { DatasetDraft } from "../../models/DatasetDraft"; -import { datasetService } from "../../services/DatasetService"; +import { datasetService, getV1Connectors } from "../../services/DatasetService"; import { obsrvError } from "../../types/ObsrvError"; import { cipherService } from "../../services/CipherService"; @@ -79,19 +79,10 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise { - return { - id: _.get(config, "id"), - connector_id: _.get(config, "connector_type"), - connector_config: _.get(config, "connector_config"), - version: "v1" - } - }) + datasetConfigs["connectors_config"] = await getV1Connectors(datasetId) datasetConfigs["transformations_config"] = _.map(transformations_config, (config) => { - console.log(config) const section: any = _.get(config, "metadata.section") || _.get(config, "transformation_function.category"); return { field_key: _.get(config, "field_key"), @@ -105,7 +96,9 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise) => { let draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) - defaultConfigs = _.omit(defaultConfigs, ["router_config"]) + defaultConfigs = _.omit(defaultConfigs, ["router_config","dedupe_config"]) if (draftDataset?.type === 'master') { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } From 181de2b5d4a33d5fa3cd2706278c84185864972e Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 23 Aug 2024 10:57:29 +0530 Subject: [PATCH 155/235] #OBS-I146: fix: linting fix --- api-service/.eslintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/.eslintrc b/api-service/.eslintrc index b209a5f4..e7996aa9 100644 --- a/api-service/.eslintrc +++ b/api-service/.eslintrc @@ -9,7 +9,8 @@ "plugin:@typescript-eslint/eslint-recommended", "plugin:@typescript-eslint/recommended" ], - "rules": { + "rules": { + "@typescript-eslint/no-var-requires": "off", "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], "@typescript-eslint/no-explicit-any": ["off"], "@typescript-eslint/no-useless-escape": ["off"], From 53262d19bc816130f1777acbc8e6c75978cfb364 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Fri, 23 Aug 2024 11:07:49 +0530 Subject: [PATCH 156/235] #OBS-I146: fix: linting fix --- api-service/.eslintrc | 1 - api-service/src/services/fs.ts | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/.eslintrc b/api-service/.eslintrc index e7996aa9..ca799240 100644 --- a/api-service/.eslintrc +++ b/api-service/.eslintrc @@ -10,7 +10,6 @@ "plugin:@typescript-eslint/recommended" ], "rules": { - "@typescript-eslint/no-var-requires": "off", "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_" }], "@typescript-eslint/no-explicit-any": ["off"], "@typescript-eslint/no-useless-escape": ["off"], diff --git a/api-service/src/services/fs.ts b/api-service/src/services/fs.ts index 774b87b9..cfc0010b 100644 --- a/api-service/src/services/fs.ts +++ b/api-service/src/services/fs.ts @@ -8,6 +8,7 @@ export const scrapModules = (folderPath: string, .map((file) => { const { default: { name, ...others }, + /* eslint-disable @typescript-eslint/no-var-requires */ } = require(path.join(folderPath, file)) as { default: Type }; mapping.set(name, others); From aed84c48a60b3d0bcd2944199a9eb16a526af31e Mon Sep 17 00:00:00 2001 From: yashashk Date: Fri, 23 Aug 2024 11:15:44 +0530 Subject: [PATCH 157/235] #OBS-I167 : Added string or dict as type to connector_config --- command-service/src/command/dataset_command.py | 1 - command-service/src/model/db_models.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/command-service/src/command/dataset_command.py b/command-service/src/command/dataset_command.py index db7afa7d..74a0a6fe 100644 --- a/command-service/src/command/dataset_command.py +++ b/command-service/src/command/dataset_command.py @@ -103,5 +103,4 @@ def audit_live_dataset(self, command_payload: CommandPayload, ts: int): "Failed to get dataset configurations from export API, dataset_id: ", dataset_id, ) - print("Dataset record details: ", json.dumps(dataset_record)) return False diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index f77f8314..23e37659 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -79,7 +79,7 @@ class DatasourcesDraft: class DatasetConnectorConfigDraft: id: str connector_id: str - connector_config: str | None + connector_config: str | dict version: str operations_config: dict | None = None data_format: str | None = 'json' From f0ba9239f1e74f93d5f25f0d5f8223e84c2ae1cf Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Fri, 23 Aug 2024 13:03:22 +0530 Subject: [PATCH 158/235] #OBS-I143: dataset publish changes fixes --- command-service/Dockerfile | 2 +- .../flink-connector/templates/deployment.yaml | 4 +- .../helm-charts/flink-connector/values.yaml | 9 +- .../spark-connector-cron/Chart.yaml | 2 +- .../src/command/connector_command.py | 324 ++++++++---------- 5 files changed, 154 insertions(+), 187 deletions(-) diff --git a/command-service/Dockerfile b/command-service/Dockerfile index 2aa9a1b5..2ca4edef 100644 --- a/command-service/Dockerfile +++ b/command-service/Dockerfile @@ -1,7 +1,7 @@ FROM --platform=linux/amd64 python:3.12-alpine COPY --from=ubuntu /usr/local/bin /usr/local/bin -RUN apk update && apk add curl jq && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ +RUN apk update && apk add curl jq vim && curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" && chmod +x kubectl && mv kubectl /usr/local/bin/ RUN cd /tmp && curl -OL https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz \ && tar -zxvf helm-v3.13.2-linux-amd64.tar.gz \ diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 946ca29d..6f4f98b7 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -181,8 +181,8 @@ spec: sleep 30s; /opt/flink/bin/flink run -m \ {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ - /flink/connectors/{{ $jobName }}-1.0.0/{{ $jobName }}-1.0.0.jar \ - --config.file.path /data/flink/conf/flink-connector.conf \ + /flink/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ + --config.file.path /data/flink/conf/connectors-scala-config.conf \ {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ {{- end }} diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index b1c48924..ff505a84 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -14,9 +14,9 @@ commonLabels: # repository: sunbirded.azurecr.io/data-pipeline # tag: "release-5.2.0_RC1_2c615f8_12" # docker pull sunbirded.azurecr.io/sunbird-datapipeline:release-4.9.0_RC4_1 -registry: surabhi1510 -repository: flink-python-3.11 -tag: 1.0.0 +registry: sanketikahub +repository: flink-connectors +tag: 1.17.2-scala_2.12-java11 imagePullSecrets: [] ## Databases @@ -300,7 +300,8 @@ serviceMonitor: jobLabel: "app.kubernetes.io/name" port: prom -flink_jobs: [] +flink_jobs: + commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file diff --git a/command-service/helm-charts/spark-connector-cron/Chart.yaml b/command-service/helm-charts/spark-connector-cron/Chart.yaml index a7a290f9..7c047bec 100644 --- a/command-service/helm-charts/spark-connector-cron/Chart.yaml +++ b/command-service/helm-charts/spark-connector-cron/Chart.yaml @@ -7,5 +7,5 @@ dependencies: description: A production-ready Helm chart base template maintainers: - name: NimbusHub.in -name: spark-submit-cron +name: spark-connector-cron version: 0.1.0 diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 7d46e64c..9698dcc6 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -37,18 +37,16 @@ def execute(self, command_payload: CommandPayload, action: Action): return result def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): - result = None - namespaces = set() - for job_type, config in self.connector_job_config.items(): - namespaces.add(config["namespace"]) - for namespace in namespaces: - self._stop_connector_jobs(dataset_id, active_connectors, is_masterdata, namespace) + result = None + self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"]) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) - return result + return result - def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata, namespace): + def _stop_connector_jobs(self, is_masterdata, namespace): + print(f"Uninstalling jobs for {namespace}..") managed_releases = [] + base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] connector_jar_config = self.config.find("connector_job") masterdata_jar_config = self.config.find("masterdata_job") for connector_type in connector_jar_config: @@ -67,14 +65,14 @@ def _stop_connector_jobs(self, dataset_id, active_connectors, is_masterdata, nam jobs = helm_ls_result.stdout.decode() for job in jobs.splitlines()[1:]: release_name = job.split()[0] - if release_name in managed_releases: + if base_helm_chart in job: print("Uninstalling job {0}".format(release_name)) helm_uninstall_cmd = [ "helm", "uninstall", release_name, "--namespace", - self.connector_job_ns, + namespace, ] helm_uninstall_result = subprocess.run( helm_uninstall_cmd, @@ -116,7 +114,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): release_name = connector_instance.connector_id runtime = connector_instance.connector_runtime namespace = self.connector_job_config["flink"]["namespace"] - + job_name = release_name.replace(".", "-") helm_ls_cmd = ["helm", "ls", "--namespace", namespace] helm_ls_result = subprocess.run( @@ -126,10 +124,10 @@ def _perform_flink_install(self, dataset_id, connector_instance): if helm_ls_result.returncode == 0: jobs = helm_ls_result.stdout.decode() print(jobs) - deployment_exists = any(release_name in line for line in jobs.splitlines()[1:]) + deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) if deployment_exists: - restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={release_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={release_name}-taskmanager --namespace {namespace}".format( - namespace=namespace, release_name=release_name + restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( + namespace=namespace, job_name=job_name ) print("Restart command: ", restart_cmd) # Run the helm command @@ -140,7 +138,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): shell=True, ) if helm_install_result.returncode == 0: - print(f"Job {release_name} re-deployment succeeded...") + print(f"Job {job_name} restart succeeded...") else: err = True return ActionResponse( @@ -155,50 +153,56 @@ def _perform_flink_install(self, dataset_id, connector_instance): return result else: - connector_source = json.loads(connector_instance.connector_source) - flink_jobs = { - "kafka-connector": { + if self._get_live_instances(runtime="flink", connector_instance=connector_instance): + connector_source = json.loads(connector_instance.connector_source) + flink_jobs = dict() + flink_jobs[release_name] = { "enabled": "true", - "job_classname": connector_source.get('main_class') + "source": connector_source.get("source"), + "main_program": connector_source.get("main_program") } - } - set_json_value = json.dumps(flink_jobs) - print("Kafka connector: ", set_json_value) - helm_install_cmd = [ - "helm", - "upgrade", - "--install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", - "--namespace", - namespace, - "--create-namespace", - "--set-json", - f"flink_jobs={set_json_value}" - ] - - print(" ".join(helm_install_cmd)) + + set_json_value = json.dumps(flink_jobs) + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + job_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["flink"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set-json", + f"flink_jobs={set_json_value.replace(" ", "")}" + ] + + print(" ".join(helm_install_cmd)) - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} deployment succeeded...") - else: - err = True - result = ActionResponse( - status="ERROR", - status_code=500, - error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", - ) - print( - f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) + + print(helm_install_result) + + if helm_install_result.returncode == 0: + print(f"Job {job_name} deployment succeeded...") + else: + err = True + result = ActionResponse( + status="ERROR", + status_code=500, + error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", + ) + print( + f"Error installing job {job_name}: {helm_install_result.stderr.decode()}" + ) - if err is None: - result = ActionResponse(status="OK", status_code=200) + if err is None: + result = ActionResponse(status="OK", status_code=200) - return result + return result + else: + self._stop_connector_jobs(is_masterdata=False, namespace="flink") else: print(f"Error checking Flink deployments: {helm_ls_result.stderr.decode()}") return ActionResponse( @@ -221,113 +225,48 @@ def _perform_spark_install(self, dataset_id, connector_instance): "Yearly": "0 0 1 1 *" # Runs at midnight on January 1st each year } - # Define namespace namespace = self.connector_job_config["spark"]["namespace"] - - # Check if the job already exists - helm_ls_cmd = ["helm", "ls", "--namespace", namespace] - helm_ls_result = subprocess.run(helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - if helm_ls_result.returncode == 0: - jobs = helm_ls_result.stdout.decode() - job_exists = any(release_name in line for line in jobs.splitlines()[1:]) - - if job_exists: - if self._is_dataset_retired(dataset_id): - print("Dataset is retired. Uninstalling existing cron job.") - self._stop_connector_jobs(dataset_id, [], False, namespace) - else: - print("Updating existing job") - - helm_install_cmd = [ - "helm", - "upgrade", - "--install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", - "--namespace", - namespace, - "--create-namespace", - "--set", - "technology={}".format(connector_instance.technology), - "--set", - "instance_id={}".format(release_name), - "--set", - "connector_source={}".format(connector_source["source"]), - "--set", - "main_class={}".format(connector_source["main_class"]), - "--set", - "main_file={}".format(connector_source["main_program"]), - "--set", - "cronSchedule={}".format(schedule_configs[schedule]) - ] - - print(" ".join(helm_install_cmd)) - - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} update succeeded...") - result = ActionResponse(status="OK", status_code=200) - else: - err = True - result = ActionResponse( - status="ERROR", - status_code=500, - error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", - ) - print(f"Error updating job {release_name}: {helm_install_result.stderr.decode()}") - else: - print("Installing new job") - - helm_install_cmd = [ - "helm", - "install", - release_name, - f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", - "--namespace", - namespace, - "--create-namespace", - "--set", - "technology={}".format(connector_instance.technology), - "--set", - "instance_id={}".format(release_name), - "--set", - "connector_source={}".format(connector_source["source"]), - "--set", - "main_class={}".format(connector_source["main_class"]), - "--set", - "main_file={}".format(connector_source["main_program"]), - "--set", - "cronSchedule={}".format(schedule_configs[schedule]) - ] - - print(" ".join(helm_install_cmd)) - - helm_install_result = subprocess.run( - helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - if helm_install_result.returncode == 0: - print(f"Job {release_name} installation succeeded...") - result = ActionResponse(status="OK", status_code=200) - else: - err = True - result = ActionResponse( - status="ERROR", - status_code=500, - error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", - ) - print(f"Error installing job {release_name}: {helm_install_result.stderr.decode()}") - + + helm_install_cmd = [ + "helm", + "upgrade", + "--install", + release_name, + f"""{self.config.find("helm_charts_base_dir")}/{self.connector_job_config["spark"]["base_helm_chart"]}""", + "--namespace", + namespace, + "--create-namespace", + "--set", + "technology={}".format(connector_instance.technology), + "--set", + "instance_id={}".format(release_name), + "--set", + "connector_source={}".format(connector_source["source"]), + "--set", + "main_class={}".format(connector_source["main_class"]), + "--set", + "main_file={}".format(connector_source["main_program"]), + "--set", + "cronSchedule={}".format(schedule_configs[schedule]) + ] + + print(" ".join(helm_install_cmd)) + + helm_install_result = subprocess.run( + helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if helm_install_result.returncode == 0: + print(f"Job {release_name} update succeeded...") + result = ActionResponse(status="OK", status_code=200) else: - print(f"Error listing Spark jobs: {helm_ls_result.stderr.decode()}") + err = True result = ActionResponse( status="ERROR", status_code=500, - error_message="SPARK_HELM_LIST_EXCEPTION", + error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", ) - + print(f"Error updating job {release_name}: {helm_install_result.stderr.decode()}") + if result is None: result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") @@ -348,7 +287,6 @@ def _get_connector_details(self, dataset_id): active_connectors.append(from_dict( data_class=ConnectorInstance, data=record )) - # connector_versions[connector_instance.id] = record['version'] return active_connectors @@ -366,37 +304,65 @@ def _get_masterdata_details(self, dataset_id): return is_masterdata - def _is_dataset_retired(self, dataset_id): - is_retired = False + ## TODO: check for connector_id as well + def _get_live_instances(self, runtime, connector_instance): + has_live_instances = False rows = self.db_service.execute_select_all( - f""" - SELECT status - FROM datasets - WHERE status='{DatasetStatusType.Retired.name}' AND dataset_id = '{dataset_id}' - """ - ) - for row in rows: - if row['status'] == DatasetStatusType.Retired.name: - print("Status: ",row['status']) - is_retired = True - break - - return is_retired - - def _get_live_instances(self, dataset_id, runtime, connector_instance): - active_connectors = [] - records = self.db_service.execute_select_all( f""" SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id FROM connector_instances ci JOIN connector_registry cr ON ci.connector_id = cr.id JOIN datasets d ON ci.dataset_id = d.id - WHERE cr.runtime = '{runtime}' AND ci.status = '{DatasetStatusType.Live.name}'; + WHERE cr.runtime = '{runtime}' AND ci.status = '{DatasetStatusType.Live.name}' AND ci.connector_id = '{connector_instance.connector_id}'; """ ) - - for record in records: - active_connectors.append(record['connector_instance_id']) + if len(rows) > 0: + has_live_instances = True - return active_connectors + return has_live_instances + + # def _perform_install(self, release): + # err = None + # result = None + # release_name = release["release_name"] + # helm_install_cmd = [ + # "helm", + # "upgrade", + # "--install", + # release_name, + # self.connector_job_chart_dir, + # "--namespace", + # self.connector_job_ns, + # "--create-namespace", + # "--set", + # "file.path={}".format(release["jar"]), + # "--set", + # "class.name={}".format(release["class"]), + # "--set", + # "job.name={}".format(release_name), + # "--set", + # "args={}".format(",".join(release["args"])), + # "--set", + # "schedule={}".format(release["schedule"]), + # ] + # helm_install_result = subprocess.run( + # helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + # ) + # if helm_install_result.returncode == 0: + # print(f"Job {release_name} deployment succeeded...") + # else: + # err = True + # result = ActionResponse( + # status="ERROR", + # status_code=500, + # error_message="FLINK_HELM_INSTALLATION_EXCEPTION", + # ) + # print( + # f"Error re-installing job {release_name}: {helm_install_result.stderr.decode()}" + # ) + + # if err is None: + # result = ActionResponse(status="OK", status_code=200) + + # return result From 512c4359e509eb2de7107be236bfb67b152c3bbc Mon Sep 17 00:00:00 2001 From: yashashk Date: Fri, 23 Aug 2024 13:27:42 +0530 Subject: [PATCH 159/235] #OBS-I167 : if dataset is empty return with error --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 63d751df..5ae41826 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -74,6 +74,9 @@ const readDraftDataset = async (datasetId: string, attributes: string[]): Promis const readDataset = async (datasetId: string, attributes: string[]): Promise => { const dataset = await datasetService.getDataset(datasetId, attributes, true); + if(!dataset) { + return; + } const api_version = _.get(dataset, "api_version") let datasetConfigs: any = {} const transformations_config = await datasetService.getTransformations(datasetId, ["field_key", "transformation_function", "mode", "metadata"]) From 6e350df904cadd614a9a8c506271b665f0628f5d Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Fri, 23 Aug 2024 13:40:32 +0530 Subject: [PATCH 160/235] #OBS-I143: inswert query fix --- command-service/src/command/connector_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index ae185174..1400668c 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -362,7 +362,7 @@ def build_insert_query(self, registry_meta: ConnectorRegsitryv2): datetime.now(), datetime.now(), - registry_meta.id, + registry_meta.id + "-" + registry_meta.version, registry_meta.name, registry_meta.type, registry_meta.category, From 98a7e3cdaf27ff28c953d03fc0efff7b193be42d Mon Sep 17 00:00:00 2001 From: Aniket Sakinala Date: Fri, 23 Aug 2024 18:35:25 +0530 Subject: [PATCH 161/235] Issue #OBS-I144 fix: icon data as string; check default version --- .../src/command/connector_registry.py | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index ae185174..0ebde996 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -1,9 +1,11 @@ import json import os +import base64 import tarfile import uuid import zipfile from datetime import datetime +from pathlib import Path import requests from fastapi import status @@ -184,6 +186,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: } connector_id = obj["id"].replace(" ", "-") + self.update_connector_registry(connector_id, self.metadata['metadata']['version']) registry_meta = ConnectorRegsitryv2(connector_id, obj['name'], 'source', @@ -194,7 +197,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: self.metadata['metadata']['runtime'], self.metadata['metadata']['licence'], self.metadata['metadata']['owner'], - obj['icon'], + self.load_file_bytes(obj["icon"]), 'Live', rel_path, json.dumps(source), @@ -225,7 +228,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: connector_id = ( self.metadata.get("metadata", {}).get("id", "").replace(" ", "-") ) - + self.update_connector_registry(connector_id, self.metadata['metadata']['version']) source = { "source": connector_source, "main_class": self.metadata["metadata"]["main_class"], @@ -243,7 +246,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: self.metadata['metadata']['runtime'], self.metadata['metadata']['licence'], self.metadata['metadata']['owner'], - self.metadata['metadata']['icon'], + self.load_file_bytes(self.metadata['metadata']["icon"]), 'Live', rel_path, json.dumps(source), @@ -380,6 +383,43 @@ def build_insert_query(self, registry_meta: ConnectorRegsitryv2): datetime.now(), ) return query, params + + def load_file_bytes(self, rel_path: str) -> bytes | None: + file_path = Path(self.extraction_path) + for item in file_path.glob("*/{}".format(rel_path)): + try: + with open(item, 'rb') as file: + file_content = file.read() + encoded = base64.b64encode(file_content).decode("ascii") + except IsADirectoryError: + print( + f"Connector Registry | No value for icon URL given at metadata: {rel_path}" + ) + return None + except FileNotFoundError: + print( + f"Connector Registry | No file present at indicated relative path: {rel_path}" + ) + return None + except (ValueError or TypeError) as e: + print( + f"Connector Registry | File content not byte like: {e}" + ) + return None + return encoded + + def update_connector_registry(self, _id, ver): + try: + result = self.db_service.execute_upsert( + f"UPDATE connector_registry SET status = 'Retired', updated_date = now() WHERE connector_id = %s AND status = 'Live' AND version != %s", (_id, ver) + ) + print( + f"Connector Registry | Updated {result} existing rows with connector_id: {_id} and version: {ver}" + ) + except Exception as e: + print( + f"Connector Registry | An error occurred during the execution of Query: {e}" + ) class ExtractionUtil: def extract_gz(tar_path, extract_path): From 19a975b401f64b8a26316526f8449a2684be364d Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Mon, 26 Aug 2024 12:33:13 +0530 Subject: [PATCH 162/235] #OBS-143: fix: dataset publish fixes --- .../flink-connector/templates/deployment.yaml | 3 ++ .../src/command/connector_command.py | 44 ++++++++++--------- 2 files changed, 26 insertions(+), 21 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 6f4f98b7..3da5d6ce 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -173,6 +173,9 @@ spec: image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} workingDir: /opt/flink + env: + - name: CONNECTOR_ID + value: {{ index $jobData "connector_id" }} command: - /bin/bash - -c diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 9698dcc6..7b2d2b13 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -45,16 +45,17 @@ def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): def _stop_connector_jobs(self, is_masterdata, namespace): print(f"Uninstalling jobs for {namespace}..") - managed_releases = [] base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] - connector_jar_config = self.config.find("connector_job") - masterdata_jar_config = self.config.find("masterdata_job") - for connector_type in connector_jar_config: - for release in connector_jar_config[connector_type]: - managed_releases.append(release["release_name"]) - if is_masterdata: - for release in masterdata_jar_config: - managed_releases.append(release["release_name"]) + + # managed_releases = [] + # connector_jar_config = self.config.find("connector_job") + # masterdata_jar_config = self.config.find("masterdata_job") + # for connector_type in connector_jar_config: + # for release in connector_jar_config[connector_type]: + # managed_releases.append(release["release_name"]) + # if is_masterdata: + # for release in masterdata_jar_config: + # managed_releases.append(release["release_name"]) helm_ls_cmd = ["helm", "ls", "--namespace", namespace] helm_ls_result = subprocess.run( @@ -123,7 +124,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): if helm_ls_result.returncode == 0: jobs = helm_ls_result.stdout.decode() - print(jobs) + deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) if deployment_exists: restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( @@ -156,8 +157,9 @@ def _perform_flink_install(self, dataset_id, connector_instance): if self._get_live_instances(runtime="flink", connector_instance=connector_instance): connector_source = json.loads(connector_instance.connector_source) flink_jobs = dict() - flink_jobs[release_name] = { + flink_jobs[job_name] = { "enabled": "true", + "connector_id": connector_instance.connector_id, "source": connector_source.get("source"), "main_program": connector_source.get("main_program") } @@ -176,7 +178,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): f"flink_jobs={set_json_value.replace(" ", "")}" ] - print(" ".join(helm_install_cmd)) + print("flink connector installation: ", " ".join(helm_install_cmd)) helm_install_result = subprocess.run( helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE @@ -185,7 +187,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): print(helm_install_result) if helm_install_result.returncode == 0: - print(f"Job {job_name} deployment succeeded...") + print(f"Job '{job_name}' deployment succeeded...") else: err = True result = ActionResponse( @@ -194,7 +196,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): error_message="FLINK_CONNECTOR_HELM_INSTALLATION_EXCEPTION", ) print( - f"Error installing job {job_name}: {helm_install_result.stderr.decode()}" + f"Error installing job '{job_name}': {helm_install_result.stderr.decode()}" ) if err is None: @@ -250,13 +252,13 @@ def _perform_spark_install(self, dataset_id, connector_instance): "cronSchedule={}".format(schedule_configs[schedule]) ] - print(" ".join(helm_install_cmd)) + print("spark connector installation:", " ".join(helm_install_cmd)) helm_install_result = subprocess.run( helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) if helm_install_result.returncode == 0: - print(f"Job {release_name} update succeeded...") + print(f"Job '{release_name}' update succeeded...") result = ActionResponse(status="OK", status_code=200) else: err = True @@ -265,7 +267,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): status_code=500, error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", ) - print(f"Error updating job {release_name}: {helm_install_result.stderr.decode()}") + print(f"Error updating job '{release_name}': {helm_install_result.stderr.decode()}") if result is None: result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") @@ -307,15 +309,15 @@ def _get_masterdata_details(self, dataset_id): ## TODO: check for connector_id as well def _get_live_instances(self, runtime, connector_instance): has_live_instances = False - rows = self.db_service.execute_select_all( - f""" + query = f""" SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id FROM connector_instances ci JOIN connector_registry cr ON ci.connector_id = cr.id JOIN datasets d ON ci.dataset_id = d.id - WHERE cr.runtime = '{runtime}' AND ci.status = '{DatasetStatusType.Live.name}' AND ci.connector_id = '{connector_instance.connector_id}'; + WHERE cr.runtime = %s AND ci.status = %s AND ci.connector_id = %s; """ - ) + params = (runtime, DatasetStatusType.Live.name, connector_instance.connector_id) + rows = self.db_service.execute_select_all(sql=query, params=params) if len(rows) > 0: has_live_instances = True From f98b1aa52f2cf00f8c62dad4869ce0f2ff7ee9ad Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 26 Aug 2024 13:21:41 +0530 Subject: [PATCH 163/235] #OBS-I167 : fix: removed duplicate code. --- api-service/src/services/DatasetService.ts | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 66a7edd6..9ec2012d 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -204,17 +204,9 @@ class DatasetService { draftDataset["sample_data"] = dataset_config?.mergedEvent draftDataset["validation_config"] = _.omit(_.get(dataset, "validation_config"), ["validation_mode"]) } else { - const v1connectors = await this.getConnectorsV1(draftDataset.dataset_id, ["id", "connector_type", "connector_config"]); - const modifiedV1Connectors = _.map(v1connectors, (config) => { - return { - id: _.get(config, "id"), - connector_id: _.get(config, "connector_type"), - connector_config: _.get(config, "connector_config"), - version: "v1" - } - }) + const v1connectors = await getV1Connectors(draftDataset.dataset_id); const v2connectors = await this.getConnectors(draftDataset.dataset_id, ["id", "connector_id", "connector_config", "operations_config"]); - draftDataset["connectors_config"] = _.concat(modifiedV1Connectors, v2connectors) + draftDataset["connectors_config"] = _.concat(v1connectors, v2connectors) const transformations = await this.getTransformations(draftDataset.dataset_id, ["field_key", "transformation_function", "mode"]); draftDataset["transformations_config"] = transformations } From b5e73069cc7fffd829a641ff8758ee814ae6fbad Mon Sep 17 00:00:00 2001 From: Harish Kumar Gangula Date: Tue, 27 Aug 2024 18:49:09 +0530 Subject: [PATCH 164/235] #OBS-I181 - Updated the event structure --- .../DataIngestion/DataIngestionController.ts | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 49e1ebf6..415849fe 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -29,7 +29,7 @@ const dataIn = async (req: Request, res: Response) => { try { const requestBody = req.body; const datasetId = req.params.datasetId.trim(); - + const isValidSchema = schemaValidation(requestBody, validationSchema) if (!isValidSchema?.isValid) { logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" }) @@ -68,18 +68,23 @@ const addMetadataToEvents = (datasetId: string, payload: any) => { const obsrvMeta = { syncts: now, flags: {}, timespans: {}, error: {}, source: source }; if (Array.isArray(validData)) { const payloadRef = validData.map((event: any) => { - event = _.set(event, "obsrv_meta", obsrvMeta); - event = _.set(event, "dataset", datasetId); - event = _.set(event, "msgid", mid); - return event + const payload = { + event, + "obsrv_meta": obsrvMeta, + "dataset": datasetId, + "msgid": mid + } + return payload; }) return payloadRef; } else { - _.set(validData, "msgid", mid); - _.set(validData, "obsrv_meta", obsrvMeta); - _.set(validData, "dataset", datasetId); - return validData + return ({ + "event": validData, + "obsrv_meta": obsrvMeta, + "dataset": datasetId, + "msgid": mid + }); } } From 77f20f4fbefe164deeba1bb36b513659b1c26c5b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 30 Aug 2024 19:34:41 +0530 Subject: [PATCH 165/235] #OBS-I164: added jwt token vwerification and access control to api's --- api-service/src/middlewares/jwtTokenVerify.ts | 156 ++++++++++++++++++ api-service/src/routes/Router.ts | 49 +++--- 2 files changed, 181 insertions(+), 24 deletions(-) create mode 100644 api-service/src/middlewares/jwtTokenVerify.ts diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts new file mode 100644 index 00000000..09d6da62 --- /dev/null +++ b/api-service/src/middlewares/jwtTokenVerify.ts @@ -0,0 +1,156 @@ +import { Request, Response, NextFunction } from "express"; +import jwt from "jsonwebtoken"; +import fs from "fs"; +import { ResponseHandler } from "../helpers/ResponseHandler"; +import _ from "lodash"; + +enum roles { + Admin = "admin", + DatasetManager = "dataset_manager", + Viewer = "viewer", + DatasetCreator = "dataset_creator", +} + +enum permissions { + DatasetCreate = "api.datasets.create", + DatasetUpdate = "api.datasets.update", + DatasetRead = "api.datasets.read", + DatasetList = "api.datasets.list", + DataIngest = "api.data.in", + DataOut = "api.data.out", + DataExhaust = "api.data.exhaust", + QueryTemplateCreate = "api.query.template.create", + QueryTemplateRead = "api.query.template.read", + QueryTemplateDelete = "api.query.template.delete", + QueryTemplateList = "api.query.template.list", + QueryTemplateUpdate = "api.query.template.update", + QueryTemplate = "api.query.template.query", + SchemaValidator = "api.schema.validator", + GenerateSignedUrl = "api.files.generate-url", + DatasetStatusTransition = "api.datasets.status-transition", + DatasetHealth = "api.dataset.health", + DatasetReset = "api.dataset.reset", + DatasetSchemaGenerator = "api.datasets.dataschema", + DatasetExport = "api.datasets.export", + DatasetCopy = "api.datasets.copy", + ConnectorList = "api.connectors.list", + ConnectorRead = "api.connectors.read", + DatasetImport = "api.datasets.import", + SQLQuery = "api.obsrv.data.sql-query" +} + +interface AccessControl { + [key: string]: string[]; +} + +const accessControl: AccessControl = { + [roles.Admin]: [ + permissions.DatasetRead, + permissions.DatasetList, + permissions.DatasetHealth, + permissions.DatasetReset, + permissions.QueryTemplateList, + permissions.ConnectorRead, + permissions.ConnectorList, + permissions.SQLQuery, + ], + [roles.DatasetManager]: [ + permissions.DatasetUpdate, + permissions.DatasetRead, + permissions.DatasetList, + permissions.DatasetHealth, + permissions.DatasetReset, + permissions.DatasetCopy, + permissions.QueryTemplateRead, + permissions.QueryTemplateUpdate, + permissions.QueryTemplateDelete, + permissions.QueryTemplateList, + permissions.SchemaValidator, + permissions.ConnectorRead, + permissions.ConnectorList, + permissions.SQLQuery, + ], + [roles.Viewer]: [ + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetRead, + permissions.DatasetList, + permissions.DatasetHealth, + permissions.DatasetReset, + permissions.QueryTemplateRead, + permissions.QueryTemplateList, + permissions.ConnectorRead, + permissions.ConnectorList, + permissions.SQLQuery, + ], + [roles.DatasetCreator]: [ + permissions.DatasetImport, + permissions.DataIngest, + permissions.DataOut, + permissions.DatasetCreate, + permissions.DatasetRead, + permissions.DatasetUpdate, + permissions.DatasetList, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SQLQuery, + permissions.DatasetStatusTransition, + permissions.DatasetSchemaGenerator, + ], +}; + +export default { + name: "jwt:tokenAuthorization", + handler: () => (req: Request, res: Response, next: NextFunction) => { + try { + const public_key = fs.readFileSync("public_key.pem", "utf8"); + const authHeader = req.headers.authorization; + const token = authHeader && authHeader.split(" ")[1]; + if (!token) { + ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "No token provided", + }, + req, + res + ); + } + jwt.verify(token as string, public_key, (err, decoded) => { + if (err) { + ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "Token verification failed", + }, + req, + res + ); + } + if (decoded && typeof decoded == "object") { + const action = (req as any).id; + const hasAccess = decoded.roles.some((role: string) => accessControl[role] && accessControl[role].includes(action)); + if (!hasAccess) { + ResponseHandler.errorResponse( + { + statusCode: 401, + errCode: "Unauthorized access", + message: "Access denied for the user", + }, + req, + res + ); + } + next(); + } + }); + } catch (error) { + next(error); + } + }, +}; diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 1faaafb0..fa91cac4 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,33 +29,34 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import {OperationType, telemetryAuditStart} from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; +import jwtTokenVerify from "../middlewares/jwtTokenVerify"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), dataExhaust); -router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); -router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); -router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); -router.post("/template/list", setDataToRequestObject("api.query.template.list"), listQueryTemplates); -router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); -router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); -router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); -router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); -router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), ConnectorsRead); -router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), dataIn); +router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), jwtTokenVerify.handler(), dataOut); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(),DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), jwtTokenVerify.handler(), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), jwtTokenVerify.handler(), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), jwtTokenVerify.handler(), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), jwtTokenVerify.handler(), dataExhaust); +router.post("/template/create", setDataToRequestObject("api.query.template.create"), jwtTokenVerify.handler(), createQueryTemplate); +router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), jwtTokenVerify.handler(), readQueryTemplate); +router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), jwtTokenVerify.handler(), deleteQueryTemplate); +router.post("/template/list", setDataToRequestObject("api.query.template.list"), jwtTokenVerify.handler(), listQueryTemplates); +router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), jwtTokenVerify.handler(), updateQueryTemplate); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), jwtTokenVerify.handler(), eventValidation); +router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), jwtTokenVerify.handler(), queryTemplate); +router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), GenerateSignedURL); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), DatasetStatusTansition); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), datasetReset); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DataSchemaGenerator); +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetExport); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), jwtTokenVerify.handler(), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), jwtTokenVerify.handler(), ConnectorsRead); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetImport); //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file From 6aa8a8f11d0b258768b8bce8b28d53ffbcdbedad Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Fri, 30 Aug 2024 19:37:20 +0530 Subject: [PATCH 166/235] #OBS-I164: added jwt token vwerification and access control to api's --- api-service/src/routes/Router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index fa91cac4..0c65be4a 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -59,4 +59,4 @@ router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read") router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetImport); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), jwtTokenVerify.handler(), sqlQuery); \ No newline at end of file From 9d31325666f292162f37518e65d2af81a8a263e8 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 2 Sep 2024 17:51:13 +0530 Subject: [PATCH 167/235] #OBS-I164: modified the access roles and permissions --- api-service/src/middlewares/jwtTokenVerify.ts | 79 ++++++++++++------- 1 file changed, 49 insertions(+), 30 deletions(-) diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts index 09d6da62..51601b8f 100644 --- a/api-service/src/middlewares/jwtTokenVerify.ts +++ b/api-service/src/middlewares/jwtTokenVerify.ts @@ -9,6 +9,7 @@ enum roles { DatasetManager = "dataset_manager", Viewer = "viewer", DatasetCreator = "dataset_creator", + Ingestor = "ingestor" } enum permissions { @@ -44,61 +45,79 @@ interface AccessControl { } const accessControl: AccessControl = { - [roles.Admin]: [ - permissions.DatasetRead, + [roles.Ingestor]: [ + permissions.DataIngest, + ], + [roles.Viewer]: [ permissions.DatasetList, - permissions.DatasetHealth, - permissions.DatasetReset, - permissions.QueryTemplateList, + permissions.DatasetRead, + permissions.DatasetExport, permissions.ConnectorRead, - permissions.ConnectorList, permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, ], - [roles.DatasetManager]: [ - permissions.DatasetUpdate, - permissions.DatasetRead, + [roles.DatasetCreator]: [ permissions.DatasetList, - permissions.DatasetHealth, - permissions.DatasetReset, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, permissions.DatasetCopy, + permissions.QueryTemplateCreate, permissions.QueryTemplateRead, - permissions.QueryTemplateUpdate, permissions.QueryTemplateDelete, - permissions.QueryTemplateList, + permissions.GenerateSignedUrl, permissions.SchemaValidator, + permissions.DatasetSchemaGenerator + ], + [roles.DatasetManager]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, permissions.ConnectorRead, - permissions.ConnectorList, permissions.SQLQuery, - ], - [roles.Viewer]: [ permissions.DataOut, permissions.DataExhaust, - permissions.DatasetRead, - permissions.DatasetList, - permissions.DatasetHealth, - permissions.DatasetReset, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, permissions.QueryTemplateRead, - permissions.QueryTemplateList, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition + ], + [roles.Admin]: [ + permissions.DatasetCreate, + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, permissions.ConnectorRead, - permissions.ConnectorList, permissions.SQLQuery, - ], - [roles.DatasetCreator]: [ - permissions.DatasetImport, - permissions.DataIngest, permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, permissions.DatasetCreate, - permissions.DatasetRead, permissions.DatasetUpdate, - permissions.DatasetList, permissions.DatasetCopy, permissions.QueryTemplateCreate, permissions.QueryTemplateRead, permissions.QueryTemplateDelete, permissions.GenerateSignedUrl, - permissions.SQLQuery, - permissions.DatasetStatusTransition, + permissions.SchemaValidator, permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition ], }; From 859a0cf823c6c2ac34f341e581c3220ee3dd8bbe Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 2 Sep 2024 18:27:03 +0530 Subject: [PATCH 168/235] #OBS-I164: reading public key from env file --- api-service/src/middlewares/jwtTokenVerify.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts index 51601b8f..3834e0ef 100644 --- a/api-service/src/middlewares/jwtTokenVerify.ts +++ b/api-service/src/middlewares/jwtTokenVerify.ts @@ -1,8 +1,7 @@ import { Request, Response, NextFunction } from "express"; import jwt from "jsonwebtoken"; -import fs from "fs"; import { ResponseHandler } from "../helpers/ResponseHandler"; -import _ from "lodash"; +import { config } from "../configs/Config"; enum roles { Admin = "admin", @@ -125,7 +124,7 @@ export default { name: "jwt:tokenAuthorization", handler: () => (req: Request, res: Response, next: NextFunction) => { try { - const public_key = fs.readFileSync("public_key.pem", "utf8"); + const public_key = config.jwt_public_key; const authHeader = req.headers.authorization; const token = authHeader && authHeader.split(" ")[1]; if (!token) { From eebc2f93a4b931b825fa5f3ce78eefb75be7b53a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Sep 2024 10:34:07 +0530 Subject: [PATCH 169/235] #OBS-I186 : fix: dataset metrics api --- .../QueryWrapper/NativeQueryWrapper.ts | 21 ++++++++++++++ .../nativeQueryValidationSchema.json | 12 ++++++++ api-service/src/routes/Router.ts | 28 ++++++++++--------- 3 files changed, 48 insertions(+), 13 deletions(-) create mode 100644 api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts create mode 100644 api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json diff --git a/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts new file mode 100644 index 00000000..c6ea4573 --- /dev/null +++ b/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts @@ -0,0 +1,21 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { executeNativeQuery } from "../../connections/druidConnection"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import vaidationSchema from "./nativeQueryValidationSchema.json" +import { schemaValidation } from "../../services/ValidationService"; +import logger from "../../logger"; +import { obsrvError } from "../../types/ObsrvError"; + +const nativeQuery = async (req: Request, res: Response) => { + const isValidSchema = schemaValidation(req.body, vaidationSchema); + if (!isValidSchema?.isValid) { + logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) + throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + } + const query = _.get(req, ["body", "query", "query"]); + const response = await executeNativeQuery(query); + ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); +} + +export default nativeQuery; \ No newline at end of file diff --git a/api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json b/api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json new file mode 100644 index 00000000..ae17e302 --- /dev/null +++ b/api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "query": { + "type": "object", + "nullable": true + } + }, + "required": [ + "query" + ] +} \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 1faaafb0..14dc40ff 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -27,35 +27,37 @@ import DatasetCopy from "../controllers/DatasetCopy/DatasetCopy"; import ConnectorsList from "../controllers/ConnectorsList/ConnectorsList"; import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; -import {OperationType, telemetryAuditStart} from "../services/telemetry"; +import { OperationType, telemetryAuditStart } from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; +import nativeQuery from "../controllers/QueryWrapper/NativeQueryWrapper"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), dataIn); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({ action: telemetryActions.createDataset, operationType: OperationType.CREATE }), dataIn); router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), dataExhaust); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.createDataset, operationType: OperationType.CREATE }), DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.updateDataset, operationType: OperationType.UPDATE }), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.readDataset, operationType: OperationType.GET }), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.listDatasets, operationType: OperationType.LIST }), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.datasetExhaust, operationType: OperationType.GET }), dataExhaust); router.post("/template/create", setDataToRequestObject("api.query.template.create"), createQueryTemplate); router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), readQueryTemplate); router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), deleteQueryTemplate); router.post("/template/list", setDataToRequestObject("api.query.template.list"), listQueryTemplates); router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), eventValidation); router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), queryTemplate); router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), DatasetStatusTansition); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.createTransformation, operationType: OperationType.CREATE }), DatasetStatusTansition); router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), datasetHealth); router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), datasetReset); router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), DataSchemaGenerator); router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), ConnectorsRead); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.copyDataset, operationType: OperationType.CREATE }), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.listConnectors, operationType: OperationType.GET }), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({ action: telemetryActions.readConnectors, operationType: OperationType.GET }), ConnectorsRead); router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), DatasetImport); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); \ No newline at end of file +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); +router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), nativeQuery) \ No newline at end of file From 38bcad9a515236bcb87eb6df3eff32f1feb50d48 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Sep 2024 10:35:29 +0530 Subject: [PATCH 170/235] #OBS-I185 : fix: removed duplicate code. --- .../src/connections/druidConnection.ts | 2 +- .../DataIngestion/DataIngestionController.ts | 23 +++----- .../controllers/DataOut/DataOutController.ts | 59 ++++++++----------- api-service/src/services/DatasetService.ts | 4 +- api-service/src/services/TableGenerator.ts | 18 +++--- .../src/command/connector_command.py | 2 +- command-service/src/command/db_command.py | 4 +- command-service/src/command/druid_command.py | 2 +- 8 files changed, 46 insertions(+), 68 deletions(-) diff --git a/api-service/src/connections/druidConnection.ts b/api-service/src/connections/druidConnection.ts index 7ca7b2dc..a25e40d8 100644 --- a/api-service/src/connections/druidConnection.ts +++ b/api-service/src/connections/druidConnection.ts @@ -3,7 +3,7 @@ import * as _ from "lodash"; import { config } from "../configs/Config"; const druidPort = _.get(config, "query_api.druid.port"); const druidHost = _.get(config, "query_api.druid.host"); -const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; +export const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; const sqlQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.sql_query_path}`; export const executeNativeQuery = async (payload: any) => { diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 415849fe..69785067 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -26,8 +26,8 @@ const apiId = "api.data.in"; const errorCode = "DATASET_UPDATE_FAILURE" const dataIn = async (req: Request, res: Response) => { - try { - const requestBody = req.body; + + const requestBody = req.body; const datasetId = req.params.datasetId.trim(); const isValidSchema = schemaValidation(requestBody, validationSchema) @@ -35,29 +35,20 @@ const dataIn = async (req: Request, res: Response) => { logger.error({ apiId, message: isValidSchema?.message, code: "DATA_INGESTION_INVALID_INPUT" }) return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_INGESTION_INVALID_INPUT" }, req, res); } - const dataset = await datasetService.getDataset(datasetId, ["id"], true) + const dataset = await datasetService.getDataset(datasetId, ["id", "entry_topic", "api_version", "dataset_config"], true) if (!dataset) { logger.error({ apiId, message: `Dataset with id ${datasetId} not found in live table`, code: "DATASET_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.datasetNotFound, req, res); } - const entryTopic = _.get(dataset, "dataValues.dataset_config.entry_topic") + const { entry_topic, dataset_config, api_version } = dataset + const entryTopic = api_version !== "v2" ? _.get(dataset_config, "entry_topic") : entry_topic if (!entryTopic) { logger.error({ apiId, message: "Entry topic not found", code: "TOPIC_NOT_FOUND" }) return ResponseHandler.errorResponse(errorObject.topicNotFound, req, res); } - await send(addMetadataToEvents(datasetId, requestBody), _.get(dataset, "dataValues.dataset_config.entry_topic")) + await send(addMetadataToEvents(datasetId, requestBody), entryTopic) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Data ingested successfully" } }); - } - catch (err: any) { - const code = _.get(err, "code") || errorCode - logger.error({ ...err, apiId, code }) - let errorMessage = err; - const statusCode = _.get(err, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: "DATA_INGESTION_FAILED", message: "Failed to ingest data" } - } - ResponseHandler.errorResponse(errorMessage, req, res); - } + } const addMetadataToEvents = (datasetId: string, payload: any) => { diff --git a/api-service/src/controllers/DataOut/DataOutController.ts b/api-service/src/controllers/DataOut/DataOutController.ts index a61bf545..dd6f93b6 100644 --- a/api-service/src/controllers/DataOut/DataOutController.ts +++ b/api-service/src/controllers/DataOut/DataOutController.ts @@ -12,44 +12,33 @@ const dataOut = async (req: Request, res: Response) => { const datasetId = req.params?.datasetId; const requestBody = req.body; const msgid = _.get(req, "body.params.msgid"); - try { - const isValidSchema = schemaValidation(requestBody, validationSchema); - if (!isValidSchema?.isValid) { - logger.error({ apiId, datasetId, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) - return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); - } - const isValidQuery: any = await validateQuery(req.body, datasetId); - const query = _.get(req, "body.query", "") - - if (isValidQuery === true && _.isObject(query)) { - const result = await executeNativeQuery(query); - logger.info({ apiId, msgid, requestBody, datasetId, message: "Native query executed successfully" }) - return ResponseHandler.successResponse(req, res, { - status: 200, data: result?.data - }); - } + const isValidSchema = schemaValidation(requestBody, validationSchema); + if (!isValidSchema?.isValid) { + logger.error({ apiId, datasetId, msgid, requestBody, message: isValidSchema?.message, code: "DATA_OUT_INVALID_INPUT" }) + return ResponseHandler.errorResponse({ message: isValidSchema?.message, statusCode: 400, errCode: "BAD_REQUEST", code: "DATA_OUT_INVALID_INPUT" }, req, res); + } + const isValidQuery: any = await validateQuery(req.body, datasetId); + const query = _.get(req, "body.query", "") - if (isValidQuery === true && _.isString(query)) { - const result = await executeSqlQuery({ query }) - logger.info({ apiId, msgid, requestBody, datasetId, message: "SQL query executed successfully" }) - return ResponseHandler.successResponse(req, res, { - status: 200, data: result?.data - }); - } + if (isValidQuery === true && _.isObject(query)) { + const result = await executeNativeQuery(query); + logger.info({ apiId, msgid, requestBody, datasetId, message: "Native query executed successfully" }) + return ResponseHandler.successResponse(req, res, { + status: 200, data: result?.data + }); + } - else { - logger.error({ apiId, msgid, requestBody, datasetId, message: isValidQuery?.message, code: isValidQuery?.code }) - return ResponseHandler.errorResponse({ message: isValidQuery?.message, statusCode: isValidQuery?.statusCode, errCode: isValidQuery?.errCode, code: isValidQuery?.code }, req, res); - } + if (isValidQuery === true && _.isString(query)) { + const result = await executeSqlQuery({ query }) + logger.info({ apiId, msgid, requestBody, datasetId, message: "SQL query executed successfully" }) + return ResponseHandler.successResponse(req, res, { + status: 200, data: result?.data + }); } - catch (err: any) { - logger.error({ ...err, apiId, code: err?.code || "INTERNAL_SERVER_ERROR" }) - let errorMessage = err; - const statusCode = _.get(err, "statusCode") - if (!statusCode || statusCode == 500) { - errorMessage = { code: "INTERNAL_SERVER_ERROR", message: "Unable to process the query." } - } - ResponseHandler.errorResponse(errorMessage, req, res); + + else { + logger.error({ apiId, msgid, requestBody, datasetId, message: isValidQuery?.message, code: isValidQuery?.code }) + return ResponseHandler.errorResponse({ message: isValidQuery?.message, statusCode: isValidQuery?.statusCode, errCode: isValidQuery?.errCode, code: isValidQuery?.code }, req, res); } } diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 9ec2012d..2e800212 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -315,7 +315,7 @@ class DatasetService { await transaction.rollback() throw obsrvError(draftDataset.id, "FAILED_TO_PUBLISH_DATASET", err.message, "SERVER_ERROR", 500, err); } - await executeCommand(draftDataset.id, "PUBLISH_DATASET"); + await executeCommand(draftDataset.dataset_id, "PUBLISH_DATASET"); } @@ -354,7 +354,7 @@ class DatasetService { return { id: _.join([datasource, type], "_"), datasource: draftDataset.dataset_id, - dataset_id: draftDataset.dataset_id, + dataset_id: draftDataset.id, datasource_ref: datasource, type } diff --git a/api-service/src/services/TableGenerator.ts b/api-service/src/services/TableGenerator.ts index eb3065ae..e22714f6 100644 --- a/api-service/src/services/TableGenerator.ts +++ b/api-service/src/services/TableGenerator.ts @@ -143,16 +143,14 @@ class TableGenerator extends BaseTableGenerator { } private getDruidFlattenSpec = (allFields: Record) => { - return _.union( - _.map(allFields, (field) => { - return { - type: "path", - expr: field.expr, - name: field.name - } - }), - rawIngestionSpecDefaults.flattenSpec - ) + const allfields = _.map(allFields, (field) => { + return { + type: "path", + expr: field.expr, + name: field.name + } + }); + return _.uniqBy([...allfields, ...rawIngestionSpecDefaults.flattenSpec], "name") } getHudiIngestionSpecForCreate = (dataset: Record, allFields: Record[], datasourceRef: string) => { diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 7b2d2b13..b18c1c33 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -175,7 +175,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): namespace, "--create-namespace", "--set-json", - f"flink_jobs={set_json_value.replace(" ", "")}" + f"""flink_jobs={set_json_value.replace(" ", "")}""" ] print("flink connector installation: ", " ".join(helm_install_cmd)) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 4cb62e2c..89db3392 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -176,7 +176,7 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): draft_datasource.datasource, dataset_id, draft_datasource.datasource_ref, - json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + json.dumps(draft_datasource.ingestion_spec), draft_datasource.type, json.dumps(draft_datasource.retention_period).replace("'", "''"), json.dumps(draft_datasource.archival_policy).replace("'", "''"), @@ -191,7 +191,7 @@ def _insert_datasource_record(self, dataset_id, draft_dataset_id): json.dumps(draft_datasource.metadata).replace("'", "''"), draft_datasource.datasource, - json.dumps(draft_datasource.ingestion_spec).replace("'", "''"), + json.dumps(draft_datasource.ingestion_spec), draft_datasource.type, json.dumps(draft_datasource.retention_period).replace("'", "''"), json.dumps(draft_datasource.archival_policy).replace("'", "''"), diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py index 901b18ad..f43222b4 100644 --- a/command-service/src/command/druid_command.py +++ b/command-service/src/command/druid_command.py @@ -35,7 +35,7 @@ def _submit_ingestion_task(self, dataset_id): f"Invoking SUBMIT_INGESTION_TASKS command for dataset_id {dataset_id}..." ) for record in datasources_records: - if record["dataset_type"] == "dataset" and record["type"] == "druid": + if record["dataset_type"] == "event" and record["type"] == "druid": print(f"Submitting ingestion task for datasource ...") ingestion_spec = json.dumps(record["ingestion_spec"]) response = self.http_service.post( From 2d0afbec43da18f9df122986c6f26bda64c8c503 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 10:52:24 +0530 Subject: [PATCH 171/235] #OBS-I164: modified public key variable in config --- api-service/src/configs/Config.ts | 5 +++-- api-service/src/middlewares/jwtTokenVerify.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 38fe624c..6a28dc37 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -4,7 +4,7 @@ const env = process.env.system_env || "local" export const config = { "env": env, - "api_port": process.env.api_port || 3000, + "api_port": process.env.api_port || 3005, "body_parser_limit": process.env.body_parser_limit || "100mb", "version": "1.0", "query_api": { @@ -112,5 +112,6 @@ export const config = { "dialect": process.env.dialet || "postgres", "url": process.env.grafana_url || "http://localhost:8000", "access_token": process.env.grafana_token || "" - } + }, + "user_token_public_key": process.env.user_token_public_key || "", } diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts index 3834e0ef..99823112 100644 --- a/api-service/src/middlewares/jwtTokenVerify.ts +++ b/api-service/src/middlewares/jwtTokenVerify.ts @@ -124,7 +124,7 @@ export default { name: "jwt:tokenAuthorization", handler: () => (req: Request, res: Response, next: NextFunction) => { try { - const public_key = config.jwt_public_key; + const public_key = config.user_token_public_key; const authHeader = req.headers.authorization; const token = authHeader && authHeader.split(" ")[1]; if (!token) { From 06525852836e9f2e43b03725fd933d263f0070d4 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Tue, 3 Sep 2024 13:17:19 +0530 Subject: [PATCH 172/235] flink connector helm chart updates --- .../flink-connector/templates/deployment.yaml | 60 ++++++----------- .../helm-charts/flink-connector/values.yaml | 66 ++----------------- 2 files changed, 27 insertions(+), 99 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 3da5d6ce..41405e4b 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -49,7 +49,7 @@ spec: containers: - name: {{ $jobName }}-taskmanager image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} workingDir: {{ .Values.taskmanager.flink_work_dir }} args: ["taskmanager"] env: @@ -69,33 +69,14 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - # - name: flink-config-volume - # mountPath: /data/flink/conf/connectors-scala-config.conf - # subPath: connectors-scala-config.conf - # - name: flink-config-volume - # mountPath: /data/flink/conf/connectors-python-config.conf - # subPath: connectors-python-config.yaml - # - name: flink-config-volume - # mountPath: /opt/flink/conf/flink-conf.yaml - # subPath: flink-conf.yaml - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties volumes: - # - name: flink-config-volume - # configMap: - # name: flink-connector-conf - # items: - # - key: connectors-scala-config.conf - # path: connectors-scala-config.conf - # - key: connectors-python-config.yaml - # path: connectors-python-config.yaml - name: flink-common-volume configMap: name: {{ $jobName }}-config items: - # - key: flink-conf - # path: flink-conf.yaml - key: log4j_console_properties path: log4j-console.properties {{ $component := "jobmanager" }} @@ -143,7 +124,7 @@ spec: containers: - name: {{ $jobName }}-jobmanager image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} workingDir: /opt/flink # command: ["/bin/sh", "-c"] args: ["jobmanager"] @@ -157,34 +138,38 @@ spec: metrics.reporter.prom.host: {{ $jobName }}-jobmanager metrics.reporter.prom.port: 9250 volumeMounts: - - name: flink-config-volume + - name: connector-config-volume mountPath: /data/flink/conf/connectors-scala-config.conf subPath: connectors-scala-config.conf - - name: flink-config-volume - mountPath: /data/flink/conf/connectors-python-config.conf - subPath: connectors-python-config.conf + - name: connector-config-volume + mountPath: /data/flink/conf/connectors-python-config.yaml + subPath: connectors-python-config.yaml - name: data - mountPath: /flink/connectors + mountPath: /data/connectors - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties - name: {{ $jobName }}-job-submit image: {{ include "base.image.flink" (dict "context" $ "scope" $jobData) }} - imagePullPolicy: {{ default .Values.imagePullPolicy "Always" }} + imagePullPolicy: {{ default "Always" .Values.imagePullPolicy }} workingDir: /opt/flink env: - name: CONNECTOR_ID value: {{ index $jobData "connector_id" }} + - name: CONFIG_PATH + value: /data/flink/conf/ + - name: CONFIG_FILE + value: connectors-python-config.yaml command: - /bin/bash - -c - | - /usr/bin/python3.11 /data/flink/connectors/connectors-init/connector.py; + /usr/bin/python3 /data/connectors-init/connector.py; sleep 30s; - /opt/flink/bin/flink run -m \ + /opt/flink/bin/flink run -d -m \ {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ - /flink/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ + /data/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ --config.file.path /data/flink/conf/connectors-scala-config.conf \ {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ @@ -198,7 +183,7 @@ spec: {{- if eq .Values.checkpoint_store_type "gcs" }} "-Dgoogle.cloud.auth.service.account.enable=true" \ {{- end }} - ; + ; sleep 10s; echo "Job submitted"; tail -f /dev/null; ports: {{- range .Values.service.ports }} - name: {{ .name }} @@ -208,23 +193,21 @@ spec: securityContext: {{- toYaml .Values.securityContext | nindent 12 }} volumeMounts: - - name: flink-config-volume + - name: connector-config-volume mountPath: /data/flink/conf/connectors-scala-config.conf subPath: connectors-scala-config.conf - - name: flink-config-volume + - name: connector-config-volume mountPath: /data/flink/conf/connectors-python-config.yaml subPath: connectors-python-config.yaml - name: data - mountPath: /flink/connectors + mountPath: /data/connectors - name: flink-common-volume mountPath: /opt/flink/conf/log4j-console.properties subPath: log4j-console.properties - - volumes: - - name: flink-config-volume + - name: connector-config-volume configMap: - name: flink-connector-conf + name: connectors-config items: - key: connectors-scala-config.conf path: connectors-scala-config.conf @@ -248,7 +231,6 @@ spec: resources: requests: storage: 2Gi - storageClassName: "standard" {{- end }} {{- end}} diff --git a/command-service/helm-charts/flink-connector/values.yaml b/command-service/helm-charts/flink-connector/values.yaml index ff505a84..dd08dafa 100644 --- a/command-service/helm-charts/flink-connector/values.yaml +++ b/command-service/helm-charts/flink-connector/values.yaml @@ -18,6 +18,7 @@ registry: sanketikahub repository: flink-connectors tag: 1.17.2-scala_2.12-java11 imagePullSecrets: [] +imagePullPolicy: IfNotPresent ## Databases global: @@ -205,65 +206,6 @@ log4j_console_properties: | logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline logger.netty.level = OFF -baseconfig: | - kafka { - broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" - producer.broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" - consumer.broker-servers = "{{ .Values.global.kafka.host }}:{{ .Values.global.kafka.port }}" - zookeeper = "{{ .Values.global.zookeeper.host }}:{{ .Values.global.zookeeper.port }}" - producer { - max-request-size = 1572864 - batch.size = 98304 - linger.ms = 10 - compression = "snappy" - } - output.system.event.topic = ${job.env}".system.events" - - output.failed.topic = ${job.env}".failed" - } - job { - env = "{{ .Values.global.env }}" - enable.distributed.checkpointing = {{ .Values.checkpointing.enabled }} - statebackend { - base.url = "{{ .Values.checkpointing.statebackend }}" - } - } - - task { - parallelism = 1 - consumer.parallelism = 1 - checkpointing.interval = 10000 - checkpointing.pause.between.seconds = 10000 - restart-strategy.attempts = 3 - restart-strategy.delay = 30000 # in milli-seconds - } - - redis.connection.timeout = 100 - redis { - host = "{{ .Values.global.redis_dedup.host }}" - port = {{ .Values.global.redis_dedup.port }} - } - - redis-meta { - host = "{{ .Values.global.redis_denorm.host }}" - port = {{ .Values.global.redis_denorm.port }} - } - - postgres { - host = "{{ .Values.global.postgresql.host }}" - port = "{{ .Values.global.postgresql.port }}" - maxConnections = "{{ .Values.postgres.max_connections }}" - sslMode = "{{ .Values.postgres.sslmode }}" - user = "{{ .Values.global.postgresql.obsrv.user }}" - password = "{{ .Values.global.postgresql.obsrv.password }}" - database = "{{ .Values.global.postgresql.obsrv.name }}" - } - - lms-cassandra { - host = "{{ .Values.global.cassandra.host }}" - port = "{{ .Values.global.cassandra.port }}" - } - flink_resources: taskmanager: resources: @@ -301,7 +243,11 @@ serviceMonitor: port: prom flink_jobs: - + "kafka-connector-1-0-0": + "enabled": "false" + "connector_id": "kafka-connector-1.0.0" + "source": "kafka-connector-1.0.0" + "main_program": "kafka-connector-1.0.0.jar" commonAnnotations: reloader.stakater.com/auto: "true" \ No newline at end of file From 9e108d8297e6a9437a388b5ad5550db817358dd4 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Tue, 3 Sep 2024 15:29:14 +0530 Subject: [PATCH 173/235] flink connector helm chart updates --- .../flink-connector/templates/deployment.yaml | 1 + .../src/command/connector_command.py | 48 +++++++++---------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/command-service/helm-charts/flink-connector/templates/deployment.yaml b/command-service/helm-charts/flink-connector/templates/deployment.yaml index 41405e4b..74fc9cca 100644 --- a/command-service/helm-charts/flink-connector/templates/deployment.yaml +++ b/command-service/helm-charts/flink-connector/templates/deployment.yaml @@ -171,6 +171,7 @@ spec: {{ $jobName }}-jobmanager.{{ include "base.namespace" . }}.svc.cluster.local:8081 \ /data/connectors/{{ index $jobData "source" }}/{{ index $jobData "main_program" }} \ --config.file.path /data/flink/conf/connectors-scala-config.conf \ + --metadata.id {{ index $jobData "connector_id" }} \ {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.global.azure_storage_account_name }}.blob.core.windows.net={{ .Values.global.azure_storage_account_key }}" \ {{- end }} diff --git a/command-service/src/command/connector_command.py b/command-service/src/command/connector_command.py index 7b2d2b13..aabb9db4 100644 --- a/command-service/src/command/connector_command.py +++ b/command-service/src/command/connector_command.py @@ -37,16 +37,16 @@ def execute(self, command_payload: CommandPayload, action: Action): return result def _deploy_connectors(self, dataset_id, active_connectors, is_masterdata): - result = None + result = None self._stop_connector_jobs(is_masterdata, self.connector_job_config["spark"]["namespace"]) result = self._install_jobs(dataset_id, active_connectors, is_masterdata) - return result + return result def _stop_connector_jobs(self, is_masterdata, namespace): print(f"Uninstalling jobs for {namespace}..") base_helm_chart = self.connector_job_config["spark"]["base_helm_chart"] - + # managed_releases = [] # connector_jar_config = self.config.find("connector_job") # masterdata_jar_config = self.config.find("masterdata_job") @@ -101,14 +101,14 @@ def _install_jobs(self, dataset_id, active_connectors, is_masterdata): f"Connector {connector.connector_id} is not supported for deployment" ) break - + # if is_masterdata: # print("Installing masterdata job") # masterdata_jar_config = self.config.find("masterdata_job") # for release in masterdata_jar_config: # result = self._perform_install(release) return result - + def _perform_flink_install(self, dataset_id, connector_instance): err = None result = None @@ -117,14 +117,14 @@ def _perform_flink_install(self, dataset_id, connector_instance): namespace = self.connector_job_config["flink"]["namespace"] job_name = release_name.replace(".", "-") helm_ls_cmd = ["helm", "ls", "--namespace", namespace] - + helm_ls_result = subprocess.run( helm_ls_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - + if helm_ls_result.returncode == 0: jobs = helm_ls_result.stdout.decode() - + deployment_exists = any(job_name in line for line in jobs.splitlines()[1:]) if deployment_exists: restart_cmd = f"kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-jobmanager --namespace {namespace} && kubectl delete pods --selector app.kubernetes.io/name=flink,component={job_name}-taskmanager --namespace {namespace}".format( @@ -148,14 +148,14 @@ def _perform_flink_install(self, dataset_id, connector_instance): error_message="FLINK_HELM_LIST_EXCEPTION", ) print(f"Error restarting pod: {helm_ls_result.stderr.decode()}") - + if err is None: result = ActionResponse(status="OK", status_code=200) return result else: if self._get_live_instances(runtime="flink", connector_instance=connector_instance): - connector_source = json.loads(connector_instance.connector_source) + connector_source = connector_instance.connector_source flink_jobs = dict() flink_jobs[job_name] = { "enabled": "true", @@ -163,7 +163,7 @@ def _perform_flink_install(self, dataset_id, connector_instance): "source": connector_source.get("source"), "main_program": connector_source.get("main_program") } - + set_json_value = json.dumps(flink_jobs) helm_install_cmd = [ "helm", @@ -175,17 +175,17 @@ def _perform_flink_install(self, dataset_id, connector_instance): namespace, "--create-namespace", "--set-json", - f"flink_jobs={set_json_value.replace(" ", "")}" + f"""flink_jobs={set_json_value.replace(" ", "")}""" ] - + print("flink connector installation: ", " ".join(helm_install_cmd)) helm_install_result = subprocess.run( helm_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE ) - + print(helm_install_result) - + if helm_install_result.returncode == 0: print(f"Job '{job_name}' deployment succeeded...") else: @@ -212,14 +212,14 @@ def _perform_flink_install(self, dataset_id, connector_instance): status_code=500, error_message="FLINK_HELM_LIST_EXCEPTION", ) - + def _perform_spark_install(self, dataset_id, connector_instance): err = None result = None release_name = connector_instance.id - connector_source = json.loads(connector_instance.connector_source) + connector_source = connector_instance.connector_source schedule = connector_instance.operations_config["schedule"] - + schedule_configs = { "Hourly": "0 * * * *", # Runs at the start of every hour "Weekly": "0 0 * * 0", # Runs at midnight every Sunday @@ -228,7 +228,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): } namespace = self.connector_job_config["spark"]["namespace"] - + helm_install_cmd = [ "helm", "upgrade", @@ -268,7 +268,7 @@ def _perform_spark_install(self, dataset_id, connector_instance): error_message="SPARK_CRON_HELM_INSTALLATION_EXCEPTION", ) print(f"Error updating job '{release_name}': {helm_install_result.stderr.decode()}") - + if result is None: result = ActionResponse(status="ERROR", status_code=500, error_message="UNKNOWN_ERROR") @@ -305,11 +305,11 @@ def _get_masterdata_details(self, dataset_id): is_masterdata = True return is_masterdata - + ## TODO: check for connector_id as well def _get_live_instances(self, runtime, connector_instance): has_live_instances = False - query = f""" + query = f""" SELECT d.id AS dataset_id, ci.id AS connector_instance_id, ci.connector_id FROM connector_instances ci JOIN connector_registry cr ON ci.connector_id = cr.id @@ -320,9 +320,9 @@ def _get_live_instances(self, runtime, connector_instance): rows = self.db_service.execute_select_all(sql=query, params=params) if len(rows) > 0: has_live_instances = True - + return has_live_instances - + # def _perform_install(self, release): # err = None # result = None From f669b5126da2986d495e723fb6e9b004a9c283d9 Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Tue, 3 Sep 2024 17:47:04 +0530 Subject: [PATCH 174/235] fix: dataset publish fixes --- .../templates/cronjob.yaml | 1 + .../src/command/connector_registry.py | 71 ++++++++++++++++++- command-service/src/model/db_models.py | 2 +- 3 files changed, 72 insertions(+), 2 deletions(-) diff --git a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml index 07b70fb2..7d35b32b 100644 --- a/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml +++ b/command-service/helm-charts/spark-connector-cron/templates/cronjob.yaml @@ -8,6 +8,7 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} spec: + concurrencyPolicy: Forbid schedule: {{ .Values.cronSchedule }} jobTemplate: spec: diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index a91f651d..862b55e2 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -8,6 +8,7 @@ from pathlib import Path import requests +import subprocess from fastapi import status from config import Config @@ -175,7 +176,9 @@ def load_ui_metadata(self, extract_out_path): def process_metadata(self, rel_path, connector_source) -> RegistryResponse: result = [] tenant = self.metadata.get("metadata", {}).get("tenant", "") - + + self.copy_connector_to_runtime(self.metadata['metadata']['runtime'], connector_source) + if tenant == "multiple": connector_objects = self.metadata["connectors"] for obj in connector_objects: @@ -210,6 +213,10 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: ) query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + if not success: return RegistryResponse( status="failure", @@ -259,6 +266,10 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: ) query, params = self.build_insert_query(registry_meta) success = self.execute_query(query, params) + + subprocess.run(["rm", "-rf", self.extraction_path]) + subprocess.run(["rm", "-rf", self.download_path]) + if not success: return RegistryResponse( status="failure", @@ -272,6 +283,7 @@ def process_metadata(self, rel_path, connector_source) -> RegistryResponse: statusCode=status.HTTP_200_OK, ) + def execute_query(self, query, params) -> bool: try: result = self.db_service.execute_upsert(sql=query, params=params) @@ -420,6 +432,63 @@ def update_connector_registry(self, _id, ver): print( f"Connector Registry | An error occurred during the execution of Query: {e}" ) + + def copy_connector_to_runtime(self, runtime: str, connector_source: str): + if runtime == "spark": + return self.copy_connector_to_spark(connector_source) + + def copy_connector_to_spark(self, connector_source: str): + print(f"Connector Registry | copying {connector_source} to spark") + ## get name of the spark pod using kubectl + spark_pod_cmd = [ + "kubectl", + "get", + "pods", + "-n", + "spark", + "-l", + "app.kubernetes.io/name=spark,app.kubernetes.io/component=master", + "-o", + "jsonpath='{.items[0].metadata.name}'", + ] + + spark_pod_result = subprocess.run(spark_pod_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + print(f"Connector Registry | spark_pod_result: {spark_pod_result}") + + if spark_pod_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to get the spark pod", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + spark_pod = spark_pod_result.stdout.decode("utf-8").replace("'", "") + print(f"Connector Registry | spark_pod: {spark_pod}") + + ## copy the connector to the spark pod under /data/connectors/{source} + source_path = os.path.join(self.extraction_path, connector_source) + copy_cmd = [ + "kubectl", + "cp", + f"{source_path}", + f"spark/{spark_pod}:/data/connectors/{connector_source}", + ] + + copy_result = subprocess.run(copy_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Connector Registry | copy_result: {copy_result}") + if copy_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to copy the connector to spark", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + return RegistryResponse( + status="success", + message="connector copied to spark successfully", + statusCode=status.HTTP_200_OK, + ) class ExtractionUtil: def extract_gz(tar_path, extract_path): diff --git a/command-service/src/model/db_models.py b/command-service/src/model/db_models.py index f77f8314..6586c52d 100644 --- a/command-service/src/model/db_models.py +++ b/command-service/src/model/db_models.py @@ -123,5 +123,5 @@ class ConnectorInstance: connector_id: str operations_config: dict connector_runtime: str - connector_source: str + connector_source: dict technology: str \ No newline at end of file From f5f366727429fe1da06583b8c0ec83639ebee9e1 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 17:53:16 +0530 Subject: [PATCH 175/235] #OBS-I164: modified public key variable in config --- api-service/src/configs/Config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 6a28dc37..7adafd0f 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -4,7 +4,7 @@ const env = process.env.system_env || "local" export const config = { "env": env, - "api_port": process.env.api_port || 3005, + "api_port": process.env.api_port || 3000, "body_parser_limit": process.env.body_parser_limit || "100mb", "version": "1.0", "query_api": { From 808ce80d3c7971cfb60c8afe1c752915964c5d92 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 3 Sep 2024 18:08:27 +0530 Subject: [PATCH 176/235] #OBS-I186 : added dataset mertric api controller and route and minor change in dataset transition api --- .../DatasetMetricsController.ts | 38 +++++ .../DatasetMetricsValidationSchema.json} | 0 .../DatasetStatusTransition.ts | 53 +++---- .../ReadyToPublishSchema.json | 130 ++++++++++++++---- .../QueryWrapper/NativeQueryWrapper.ts | 21 --- api-service/src/routes/Router.ts | 4 +- 6 files changed, 171 insertions(+), 75 deletions(-) create mode 100644 api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts rename api-service/src/controllers/{QueryWrapper/nativeQueryValidationSchema.json => DatasetMetrics/DatasetMetricsValidationSchema.json} (100%) delete mode 100644 api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts new file mode 100644 index 00000000..0e0d2193 --- /dev/null +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -0,0 +1,38 @@ +import { Request, Response } from "express"; +import _ from "lodash"; +import { executeNativeQuery } from "../../connections/druidConnection"; +import { ResponseHandler } from "../../helpers/ResponseHandler"; +import vaidationSchema from "./DatasetMetricsValidationSchema.json" +import { schemaValidation } from "../../services/ValidationService"; +import logger from "../../logger"; +import { obsrvError } from "../../types/ObsrvError"; +import axios from "axios"; +import { config } from "../../configs/Config"; + +const getBaseUrl = (url: string) => { + if (_.startsWith(url, '/prom')) return config.query_api.prometheus.url + _.replace(url, '/prom', '') +} + +const datasetMetrics = async (req: Request, res: Response) => { + const isValidSchema = schemaValidation(req.body, vaidationSchema); + if (!isValidSchema?.isValid) { + logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) + throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) + } + let { query } = req.body || {}; + let endpoint = query.url; + if (_.startsWith(endpoint, '/prom')) { + query.url = getBaseUrl(endpoint) + const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; + const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) + const data = _.get(apiResponse, 'data'); + return res.json(data); + } + else { + const query = _.get(req, ["body", "query", "body", "query"]); + const response = await executeNativeQuery(query); + ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); + } +} + +export default datasetMetrics; \ No newline at end of file diff --git a/api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json b/api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json similarity index 100% rename from api-service/src/controllers/QueryWrapper/nativeQueryValidationSchema.json rename to api-service/src/controllers/DatasetMetrics/DatasetMetricsValidationSchema.json diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 5c86d519..eb07934c 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -24,7 +24,7 @@ const allowedTransitions: Record = { } const liveDatasetActions = ["Retire", "Archive", "Purge"] -const validateRequest = (req: Request, datasetId: any) => { +const validateRequest = (req: Request, datasetId: any) => { const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) if (!isRequestValid.isValid) { throw obsrvError(datasetId, invalidRequest, isRequestValid.message, "BAD_REQUEST", 400) @@ -32,7 +32,7 @@ const validateRequest = (req: Request, datasetId: any) => { } const validateDataset = (dataset: any, datasetId: any, action: string) => { - + if (_.isEmpty(dataset)) { throw obsrvError(datasetId, datasetNotFound, `Dataset not found for dataset: ${datasetId}`, "NOT_FOUND", 404) } @@ -41,7 +41,7 @@ const validateDataset = (dataset: any, datasetId: any, action: string) => { throw obsrvError(datasetId, "DATASET_API_VERSION_MISMATCH", "Draft dataset api version is not v2. Perform a read api call with mode=edit to migrate the dataset", "BAD_REQUEST", 400) } - if(!_.includes(allowedTransitions[action], dataset.status)) { + if (!_.includes(allowedTransitions[action], dataset.status)) { throw obsrvError(datasetId, `DATASET_${_.toUpper(action)}_FAILURE`, `Transition failed for dataset: ${dataset.id} status:${dataset.status} with status transition to ${action}`, "CONFLICT", 409) } @@ -54,10 +54,10 @@ const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); validateRequest(req, dataset_id); - const dataset:Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) validateDataset(dataset, dataset_id, status); - switch(status) { + switch (status) { case "Delete": await deleteDataset(dataset); break; @@ -74,7 +74,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await archiveDataset(dataset); break; case "Purge": - await purgeDataset(dataset); + await purgeDataset(dataset); break; } @@ -94,11 +94,12 @@ const readyForPublish = async (dataset: Record) => { const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) - defaultConfigs = _.omit(defaultConfigs, ["router_config","dedupe_config"]) + defaultConfigs = _.omit(defaultConfigs, ["router_config"]) + defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); if (draftDataset?.type === 'master') { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } - _.mergeWith(draftDataset,defaultConfigs,draftDataset, (objValue, srcValue) => { + _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue) => { if (_.isBoolean(objValue) && _.isBoolean(srcValue)) { return objValue; } @@ -128,7 +129,7 @@ const readyForPublish = async (dataset: Record) => { const publishDataset = async (dataset: Record) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record - + await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) await datasetService.publishDataset(draftDataset) @@ -138,9 +139,9 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) // 1. Check if there are denorm fields and dependent master datasets are published const denormConfig = _.get(draftDataset, "denorm_config") - if(denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { + if (denormConfig && !_.isEmpty(denormConfig.denorm_fields)) { const datasetIds = _.map(denormConfig.denorm_fields, "dataset_id") - if(_.includes(datasetIds, draftDataset.id)) { + if (_.includes(datasetIds, draftDataset.id)) { throw { code: "SELF_REFERENCING_MASTER_DATA", message: `The denorm master dataset is self-referencing itself`, @@ -148,26 +149,26 @@ const validateAndUpdateDenormConfig = async (draftDataset: Record) statusCode: 409 } } - const masterDatasets = await datasetService.findDatasets({id: datasetIds, type: "master"}, ["id", "status", "dataset_config", "api_version"]) + const masterDatasets = await datasetService.findDatasets({ id: datasetIds, type: "master" }, ["id", "status", "dataset_config", "api_version"]) const masterDatasetsStatus = _.map(denormConfig.denorm_fields, (denormField) => { const md = _.find(masterDatasets, (master) => { return denormField.dataset_id === master.id }) - const datasetStatus : Record = { + const datasetStatus: Record = { dataset_id: denormField.dataset_id, exists: (md) ? true : false, - isLive: (md) ? md.status === "Live" : false, + isLive: (md) ? md.status === "Live" : false, status: (md) ? md.status : false } - if(!_.isEmpty(md)){ - if(md.api_version === "v2") - datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.cache_config.redis_db}); - else - datasetStatus["denorm_field"] = _.merge(denormField, {redis_db: md.dataset_config.redis_db}); + if (!_.isEmpty(md)) { + if (md.api_version === "v2") + datasetStatus["denorm_field"] = _.merge(denormField, { redis_db: md.dataset_config.cache_config.redis_db }); + else + datasetStatus["denorm_field"] = _.merge(denormField, { redis_db: md.dataset_config.redis_db }); } return datasetStatus; }) - const invalidMasters = _.filter(masterDatasetsStatus, {isLive: false}) - if(_.size(invalidMasters) > 0) { + const invalidMasters = _.filter(masterDatasetsStatus, { isLive: false }) + if (_.size(invalidMasters) > 0) { const invalidIds = _.map(invalidMasters, "dataset_id") throw { code: "DEPENDENT_MASTER_DATA_NOT_LIVE", @@ -193,7 +194,7 @@ const updateMasterDataConfig = async (draftDataset: Record) => { draftDataset.dataset_config = { ...dataset_config, cache_config: datasetCacheConfig } if (draftDataset.dataset_config.cache_config.redis_db === 0) { const { results }: any = await datasetService.getNextRedisDBIndex() - if(_.isEmpty(results)) { + if (_.isEmpty(results)) { throw { code: "REDIS_DB_INDEX_FETCH_FAILED", message: `Unable to fetch the redis db index for the master data`, @@ -222,14 +223,14 @@ const canRetireIfMasterDataset = async (dataset: Record) => { const liveDatasets = await datasetService.findDatasets({ status: DatasetStatus.Live }, ["dataset_config", "dataset_id"]) || [] const draftDatasets = await datasetService.findDraftDatasets({ status: [DatasetStatus.ReadyToPublish, DatasetStatus.Draft] }, ["denorm_config", "id", "status"]) || [] const allDatasets = _.union(liveDatasets, draftDatasets) - const extractDenormFields = _.map(allDatasets, function(depDataset) { - return {dataset_id: _.get(depDataset, "id"), status: _.get(depDataset, "status"), denorm_datasets: _.map(_.get(depDataset, "denorm_config.denorm_fields"), "dataset_id")} + const extractDenormFields = _.map(allDatasets, function (depDataset) { + return { dataset_id: _.get(depDataset, "id"), status: _.get(depDataset, "status"), denorm_datasets: _.map(_.get(depDataset, "denorm_config.denorm_fields"), "dataset_id") } }) - const deps = _.filter(extractDenormFields, function(depDS) { return _.includes(depDS.denorm_datasets, dataset.id)}) + const deps = _.filter(extractDenormFields, function (depDS) { return _.includes(depDS.denorm_datasets, dataset.id) }) if (_.size(deps) > 0) { const denormErrMsg = `Failed to retire dataset as it is in use. Please retire or delete dependent datasets before retiring this dataset` - throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function(o) { return _.omit(o, "denorm_datasets")})) + throw obsrvError(dataset.id, "DATASET_IN_USE", denormErrMsg, "BAD_REQUEST", 400, undefined, _.map(deps, function (o) { return _.omit(o, "denorm_datasets") })) } } } diff --git a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json index bbb67fdb..eff208cb 100644 --- a/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json +++ b/api-service/src/controllers/DatasetStatusTransition/ReadyToPublishSchema.json @@ -14,11 +14,17 @@ }, "status": { "type": "string", - "enum": ["Draft"] + "enum": [ + "Draft" + ] }, "type": { "type": "string", - "enum": ["event", "transaction", "master"] + "enum": [ + "event", + "transaction", + "master" + ] }, "name": { "type": "string", @@ -36,10 +42,16 @@ }, "mode": { "type": "string", - "enum": ["Strict", "IgnoreNewFields"] + "enum": [ + "Strict", + "IgnoreNewFields" + ] } }, - "required": ["validate", "mode"] + "required": [ + "validate", + "mode" + ] }, "extraction_config": { "type": "object", @@ -78,9 +90,15 @@ "dedup_key": { "minLength": 1 } - } + }, + "required": [ + "dedup_key", + "dedup_period" + ] }, - "required": ["drop_duplicates", "dedup_key", "dedup_period"], + "required": [ + "drop_duplicates" + ], "additionalProperties": false } }, @@ -133,9 +151,15 @@ "dedup_key": { "minLength": 1 } - } + }, + "required": [ + "dedup_key", + "dedup_period" + ] }, - "required": ["drop_duplicates", "dedup_key", "dedup_period"], + "required": [ + "drop_duplicates" + ], "additionalProperties": false }, "data_schema": { @@ -160,7 +184,10 @@ "type": "boolean" } }, - "required": ["type", "properties"], + "required": [ + "type", + "properties" + ], "additionalProperties": false }, "denorm_config": { @@ -200,17 +227,27 @@ }, "oneOf": [ { - "required": ["dataset_id", "denorm_out_field", "denorm_key"] + "required": [ + "dataset_id", + "denorm_out_field", + "denorm_key" + ] }, { - "required": ["dataset_id", "denorm_out_field", "jsonata_expr"] + "required": [ + "dataset_id", + "denorm_out_field", + "jsonata_expr" + ] } ], "additionalProperties": false } } }, - "required": ["denorm_fields"], + "required": [ + "denorm_fields" + ], "additionalProperties": false }, "router_config": { @@ -221,7 +258,9 @@ "minLength": 1 } }, - "required": ["topic"], + "required": [ + "topic" + ], "additionalProperties": false }, "dataset_config": { @@ -253,9 +292,13 @@ "default": false } }, - "required": ["olap_store_enabled", "lakehouse_enabled", "cache_enabled"], + "required": [ + "olap_store_enabled", + "lakehouse_enabled", + "cache_enabled" + ], "additionalProperties": false - }, + }, "keys_config": { "type": "object", "properties": { @@ -273,7 +316,10 @@ "minLength": 1 } }, - "required": ["data_key", "timestamp_key"], + "required": [ + "data_key", + "timestamp_key" + ], "additionalProperties": false }, "cache_config": { @@ -290,11 +336,18 @@ "type": "integer" } }, - "required": ["redis_db_host", "redis_db_port"], + "required": [ + "redis_db_host", + "redis_db_port" + ], "additionalProperties": false } }, - "required": ["indexing_config", "keys_config", "cache_config"], + "required": [ + "indexing_config", + "keys_config", + "cache_config" + ], "additionalProperties": false }, "tags": { @@ -337,7 +390,10 @@ "type": "string" } }, - "required": ["type", "expr"], + "required": [ + "type", + "expr" + ], "additionalProperties": false }, "datatype": { @@ -345,19 +401,34 @@ }, "category": { "type": "string", - "enum": ["pii", "transform", "derived"] + "enum": [ + "pii", + "transform", + "derived" + ] } }, - "required": ["type", "expr", "category"], + "required": [ + "type", + "expr", + "category" + ], "additionalProperties": false }, "mode": { "type": "string", - "enum": ["Strict", "Lenient"] + "enum": [ + "Strict", + "Lenient" + ] } }, "additionalProperties": false, - "required": ["field_key", "transformation_function", "mode"] + "required": [ + "field_key", + "transformation_function", + "mode" + ] } }, "connectors_config": { @@ -392,7 +463,12 @@ } }, "additionalProperties": false, - "required": ["id", "connector_id", "connector_config", "version"] + "required": [ + "id", + "connector_id", + "connector_config", + "version" + ] } } }, @@ -413,7 +489,9 @@ "minLength": 1 } }, - "required": ["data_key"] + "required": [ + "data_key" + ] } } } @@ -437,4 +515,4 @@ "transformations_config", "connectors_config" ] -} +} \ No newline at end of file diff --git a/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts b/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts deleted file mode 100644 index c6ea4573..00000000 --- a/api-service/src/controllers/QueryWrapper/NativeQueryWrapper.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Request, Response } from "express"; -import _ from "lodash"; -import { executeNativeQuery } from "../../connections/druidConnection"; -import { ResponseHandler } from "../../helpers/ResponseHandler"; -import vaidationSchema from "./nativeQueryValidationSchema.json" -import { schemaValidation } from "../../services/ValidationService"; -import logger from "../../logger"; -import { obsrvError } from "../../types/ObsrvError"; - -const nativeQuery = async (req: Request, res: Response) => { - const isValidSchema = schemaValidation(req.body, vaidationSchema); - if (!isValidSchema?.isValid) { - logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) - throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) - } - const query = _.get(req, ["body", "query", "query"]); - const response = await executeNativeQuery(query); - ResponseHandler.successResponse(req, res, { status: 200, data: _.get(response, "data") }); -} - -export default nativeQuery; \ No newline at end of file diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 14dc40ff..d40ed232 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,7 +29,7 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import { OperationType, telemetryAuditStart } from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; -import nativeQuery from "../controllers/QueryWrapper/NativeQueryWrapper"; +import datasetMetrics from "../controllers/DatasetMetrics/DatasetMetricsController"; export const router = express.Router(); @@ -60,4 +60,4 @@ router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), o //Wrapper Service router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), sqlQuery); -router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), nativeQuery) \ No newline at end of file +router.post("/data/metrics", setDataToRequestObject("api.data.metrics"), onRequest({ entity: Entity.Data_out }), datasetMetrics) \ No newline at end of file From 9a08837d5730c11b82b055f5ce8c6ec84a39051c Mon Sep 17 00:00:00 2001 From: SurabhiAngadi Date: Tue, 3 Sep 2024 18:35:20 +0530 Subject: [PATCH 177/235] install pip requirments if applicable --- .../src/command/connector_registry.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/command-service/src/command/connector_registry.py b/command-service/src/command/connector_registry.py index 862b55e2..ceb2cbba 100644 --- a/command-service/src/command/connector_registry.py +++ b/command-service/src/command/connector_registry.py @@ -436,6 +436,7 @@ def update_connector_registry(self, _id, ver): def copy_connector_to_runtime(self, runtime: str, connector_source: str): if runtime == "spark": return self.copy_connector_to_spark(connector_source) + def copy_connector_to_spark(self, connector_source: str): print(f"Connector Registry | copying {connector_source} to spark") @@ -483,7 +484,29 @@ def copy_connector_to_spark(self, connector_source: str): message="failed to copy the connector to spark", statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, ) - + + if self.metadata['metadata']['technology'] == "python": + pip_install_cmd = [ + "kubectl", + "exec", + f"pod/{spark_pod}", + "-n", + "spark", + "--", + "bash", + "-c", + f"pip install -r /data/connectors/{connector_source}/requirements.txt", + ] + + pip_install_result = subprocess.run(pip_install_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + print(f"Connector Registry | pip_install_result: {pip_install_result}") + if pip_install_result.returncode != 0: + return RegistryResponse( + status="failure", + message="failed to install the requirements on spark", + statusCode=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + return RegistryResponse( status="success", message="connector copied to spark successfully", From 588245bd9d447636a3e392ce0098452277e1b3d9 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 3 Sep 2024 18:42:53 +0530 Subject: [PATCH 178/235] #OBS-I186 : removed export statement --- api-service/src/connections/druidConnection.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/connections/druidConnection.ts b/api-service/src/connections/druidConnection.ts index a25e40d8..7ca7b2dc 100644 --- a/api-service/src/connections/druidConnection.ts +++ b/api-service/src/connections/druidConnection.ts @@ -3,7 +3,7 @@ import * as _ from "lodash"; import { config } from "../configs/Config"; const druidPort = _.get(config, "query_api.druid.port"); const druidHost = _.get(config, "query_api.druid.host"); -export const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; +const nativeQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.native_query_path}`; const sqlQueryEndpoint = `${druidHost}:${druidPort}${config.query_api.druid.sql_query_path}`; export const executeNativeQuery = async (payload: any) => { From 4f5e155f29fd42ffc0eb31ca703922c31f9d06a0 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 18:45:36 +0530 Subject: [PATCH 179/235] #OBS-I164: added config for option rbac verification --- api-service/src/configs/Config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 7adafd0f..81702432 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -114,4 +114,5 @@ export const config = { "access_token": process.env.grafana_token || "" }, "user_token_public_key": process.env.user_token_public_key || "", + "is_RBAC_enabled": false } From 9d16185a53b4824779431057f7e503acde9894fb Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 18:46:27 +0530 Subject: [PATCH 180/235] #OBS-I164: changed the middleware to rbac_middleware --- .../src/middlewares/RBAC_middleware.ts | 178 ++++++++++++++++++ api-service/src/middlewares/jwtTokenVerify.ts | 174 ----------------- api-service/src/routes/Router.ts | 52 ++--- 3 files changed, 204 insertions(+), 200 deletions(-) create mode 100644 api-service/src/middlewares/RBAC_middleware.ts delete mode 100644 api-service/src/middlewares/jwtTokenVerify.ts diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts new file mode 100644 index 00000000..b84b2edb --- /dev/null +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -0,0 +1,178 @@ +import { Request, Response, NextFunction } from "express"; +import jwt from "jsonwebtoken"; +import { ResponseHandler } from "../helpers/ResponseHandler"; +import { config } from "../configs/Config"; + +enum roles { + Admin = "admin", + DatasetManager = "dataset_manager", + Viewer = "viewer", + DatasetCreator = "dataset_creator", + Ingestor = "ingestor", +} + +enum permissions { + DatasetCreate = "api.datasets.create", + DatasetUpdate = "api.datasets.update", + DatasetRead = "api.datasets.read", + DatasetList = "api.datasets.list", + DataIngest = "api.data.in", + DataOut = "api.data.out", + DataExhaust = "api.data.exhaust", + QueryTemplateCreate = "api.query.template.create", + QueryTemplateRead = "api.query.template.read", + QueryTemplateDelete = "api.query.template.delete", + QueryTemplateList = "api.query.template.list", + QueryTemplateUpdate = "api.query.template.update", + QueryTemplate = "api.query.template.query", + SchemaValidator = "api.schema.validator", + GenerateSignedUrl = "api.files.generate-url", + DatasetStatusTransition = "api.datasets.status-transition", + DatasetHealth = "api.dataset.health", + DatasetReset = "api.dataset.reset", + DatasetSchemaGenerator = "api.datasets.dataschema", + DatasetExport = "api.datasets.export", + DatasetCopy = "api.datasets.copy", + ConnectorList = "api.connectors.list", + ConnectorRead = "api.connectors.read", + DatasetImport = "api.datasets.import", + SQLQuery = "api.obsrv.data.sql-query", +} + +interface AccessControl { + [key: string]: string[]; +} + +const accessControl: AccessControl = { + [roles.Ingestor]: [permissions.DataIngest], + [roles.Viewer]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + ], + [roles.DatasetCreator]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + ], + [roles.DatasetManager]: [ + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition, + ], + [roles.Admin]: [ + permissions.DatasetCreate, + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, + permissions.DatasetImport, + permissions.DatasetCreate, + permissions.DatasetUpdate, + permissions.DatasetCopy, + permissions.QueryTemplateCreate, + permissions.QueryTemplateRead, + permissions.QueryTemplateDelete, + permissions.GenerateSignedUrl, + permissions.SchemaValidator, + permissions.DatasetSchemaGenerator, + permissions.DatasetReset, + permissions.DatasetStatusTransition, + ], +}; + +export default { + name: "rbac:middleware", + handler: () => (req: Request, res: Response, next: NextFunction) => { + try { + if (config.is_RBAC_enabled === true) { + const public_key = config.user_token_public_key; + const token = req.get("x-user-token"); + if (!token) { + return ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "No token provided", + }, + req, + res + ); + } + jwt.verify(token as string, public_key, (err, decoded) => { + if (err) { + return ResponseHandler.errorResponse( + { + statusCode: 403, + errCode: "FORBIDDEN", + message: "Token verification failed", + }, + req, + res + ); + } + if (decoded && typeof decoded == "object") { + const action = (req as any).id; + const hasAccess = decoded.roles.some( + (role: string) => + accessControl[role] && accessControl[role].includes(action) + ); + if (!hasAccess) { + return ResponseHandler.errorResponse( + { + statusCode: 401, + errCode: "Unauthorized access", + message: "Access denied for the user", + }, + req, + res + ); + } + next(); + } + }); + } else { + next(); + } + } catch (error) { + next(error); + } + }, +}; diff --git a/api-service/src/middlewares/jwtTokenVerify.ts b/api-service/src/middlewares/jwtTokenVerify.ts deleted file mode 100644 index 99823112..00000000 --- a/api-service/src/middlewares/jwtTokenVerify.ts +++ /dev/null @@ -1,174 +0,0 @@ -import { Request, Response, NextFunction } from "express"; -import jwt from "jsonwebtoken"; -import { ResponseHandler } from "../helpers/ResponseHandler"; -import { config } from "../configs/Config"; - -enum roles { - Admin = "admin", - DatasetManager = "dataset_manager", - Viewer = "viewer", - DatasetCreator = "dataset_creator", - Ingestor = "ingestor" -} - -enum permissions { - DatasetCreate = "api.datasets.create", - DatasetUpdate = "api.datasets.update", - DatasetRead = "api.datasets.read", - DatasetList = "api.datasets.list", - DataIngest = "api.data.in", - DataOut = "api.data.out", - DataExhaust = "api.data.exhaust", - QueryTemplateCreate = "api.query.template.create", - QueryTemplateRead = "api.query.template.read", - QueryTemplateDelete = "api.query.template.delete", - QueryTemplateList = "api.query.template.list", - QueryTemplateUpdate = "api.query.template.update", - QueryTemplate = "api.query.template.query", - SchemaValidator = "api.schema.validator", - GenerateSignedUrl = "api.files.generate-url", - DatasetStatusTransition = "api.datasets.status-transition", - DatasetHealth = "api.dataset.health", - DatasetReset = "api.dataset.reset", - DatasetSchemaGenerator = "api.datasets.dataschema", - DatasetExport = "api.datasets.export", - DatasetCopy = "api.datasets.copy", - ConnectorList = "api.connectors.list", - ConnectorRead = "api.connectors.read", - DatasetImport = "api.datasets.import", - SQLQuery = "api.obsrv.data.sql-query" -} - -interface AccessControl { - [key: string]: string[]; -} - -const accessControl: AccessControl = { - [roles.Ingestor]: [ - permissions.DataIngest, - ], - [roles.Viewer]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - ], - [roles.DatasetCreator]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator - ], - [roles.DatasetManager]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.DatasetReset, - permissions.DatasetStatusTransition - ], - [roles.Admin]: [ - permissions.DatasetCreate, - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.DatasetReset, - permissions.DatasetStatusTransition - ], -}; - -export default { - name: "jwt:tokenAuthorization", - handler: () => (req: Request, res: Response, next: NextFunction) => { - try { - const public_key = config.user_token_public_key; - const authHeader = req.headers.authorization; - const token = authHeader && authHeader.split(" ")[1]; - if (!token) { - ResponseHandler.errorResponse( - { - statusCode: 403, - errCode: "FORBIDDEN", - message: "No token provided", - }, - req, - res - ); - } - jwt.verify(token as string, public_key, (err, decoded) => { - if (err) { - ResponseHandler.errorResponse( - { - statusCode: 403, - errCode: "FORBIDDEN", - message: "Token verification failed", - }, - req, - res - ); - } - if (decoded && typeof decoded == "object") { - const action = (req as any).id; - const hasAccess = decoded.roles.some((role: string) => accessControl[role] && accessControl[role].includes(action)); - if (!hasAccess) { - ResponseHandler.errorResponse( - { - statusCode: 401, - errCode: "Unauthorized access", - message: "Access denied for the user", - }, - req, - res - ); - } - next(); - } - }); - } catch (error) { - next(error); - } - }, -}; diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index 0c65be4a..d2e53b3b 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,34 +29,34 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import {OperationType, telemetryAuditStart} from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; -import jwtTokenVerify from "../middlewares/jwtTokenVerify"; +import rbacVerify from "../middlewares/RBAC_middleware"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), jwtTokenVerify.handler(), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(),DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), jwtTokenVerify.handler(), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), jwtTokenVerify.handler(), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), jwtTokenVerify.handler(), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), jwtTokenVerify.handler(), dataExhaust); -router.post("/template/create", setDataToRequestObject("api.query.template.create"), jwtTokenVerify.handler(), createQueryTemplate); -router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), jwtTokenVerify.handler(), readQueryTemplate); -router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), jwtTokenVerify.handler(), deleteQueryTemplate); -router.post("/template/list", setDataToRequestObject("api.query.template.list"), jwtTokenVerify.handler(), listQueryTemplates); -router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), jwtTokenVerify.handler(), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), jwtTokenVerify.handler(), eventValidation); -router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), jwtTokenVerify.handler(), queryTemplate); -router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), datasetReset); -router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DataSchemaGenerator); -router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), jwtTokenVerify.handler(), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), jwtTokenVerify.handler(), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), jwtTokenVerify.handler(), ConnectorsRead); -router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), jwtTokenVerify.handler(), DatasetImport); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), rbacVerify.handler(), dataIn); +router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), rbacVerify.handler(), dataOut); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), rbacVerify.handler(),DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), rbacVerify.handler(), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), rbacVerify.handler(), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), rbacVerify.handler(), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), rbacVerify.handler(), dataExhaust); +router.post("/template/create", setDataToRequestObject("api.query.template.create"), rbacVerify.handler(), createQueryTemplate); +router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), rbacVerify.handler(), readQueryTemplate); +router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), rbacVerify.handler(), deleteQueryTemplate); +router.post("/template/list", setDataToRequestObject("api.query.template.list"), rbacVerify.handler(), listQueryTemplates); +router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), rbacVerify.handler(), updateQueryTemplate); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), rbacVerify.handler(), eventValidation); +router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), rbacVerify.handler(), queryTemplate); +router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), GenerateSignedURL); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), rbacVerify.handler(), DatasetStatusTansition); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), datasetReset); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DataSchemaGenerator); +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DatasetExport); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), rbacVerify.handler(), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), rbacVerify.handler(), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), rbacVerify.handler(), ConnectorsRead); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DatasetImport); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), jwtTokenVerify.handler(), sqlQuery); \ No newline at end of file +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), rbacVerify.handler(), sqlQuery); \ No newline at end of file From 5d8563371372cbb3b045f44c7f187551916a9c13 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 3 Sep 2024 19:19:50 +0530 Subject: [PATCH 181/235] #OBS-I186 : Logic moved to separate function --- .../controllers/DatasetRead/DatasetRead.ts | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 5ae41826..e34a996b 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -36,19 +36,7 @@ const datasetRead = async (req: Request, res: Response) => { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } if (dataset.connectors_config) { - dataset.connectors_config = dataset?.connectors_config.map((connector: any) => { - let connector_config = _.get(connector, "connector_config") - const authMechanism = _.get(connector_config, ["authenticationMechanism"]) - if (authMechanism && authMechanism.encrypted) { - connector_config = { - ...connector_config, - authenticationMechanism: JSON.parse(cipherService.decrypt(authMechanism.encryptedValues))} - } - return { - ...connector, - connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector_config)) - } - }); + dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } @@ -74,7 +62,7 @@ const readDraftDataset = async (datasetId: string, attributes: string[]): Promis const readDataset = async (datasetId: string, attributes: string[]): Promise => { const dataset = await datasetService.getDataset(datasetId, attributes, true); - if(!dataset) { + if (!dataset) { return; } const api_version = _.get(dataset, "api_version") @@ -104,4 +92,23 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise { + return connectorsConfig.map((connector: any) => { + let connector_config = _.get(connector, "connector_config"); + const authMechanism = _.get(connector_config, ["authenticationMechanism"]); + + if (authMechanism && authMechanism.encrypted) { + connector_config = { + ...connector_config, + authenticationMechanism: JSON.parse(cipherService.decrypt(authMechanism.encryptedValues)) + }; + } + + return { + ...connector, + connector_config: _.isObject(connector_config) ? connector_config : JSON.parse(cipherService.decrypt(connector_config)) + }; + }); +} + export default datasetRead; \ No newline at end of file From ac09f850a5d11ded6b601fa7136a165082c60764 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 19:22:18 +0530 Subject: [PATCH 182/235] #OBS-I164: changed import name --- api-service/src/routes/Router.ts | 52 ++++++++++++++++---------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/api-service/src/routes/Router.ts b/api-service/src/routes/Router.ts index d2e53b3b..bf66e34d 100644 --- a/api-service/src/routes/Router.ts +++ b/api-service/src/routes/Router.ts @@ -29,34 +29,34 @@ import ConnectorsRead from "../controllers/ConnectorsRead/ConnectorsRead"; import DatasetImport from "../controllers/DatasetImport/DatasetImport"; import {OperationType, telemetryAuditStart} from "../services/telemetry"; import telemetryActions from "../telemetry/telemetryActions"; -import rbacVerify from "../middlewares/RBAC_middleware"; +import checkRBAC from "../middlewares/RBAC_middleware"; export const router = express.Router(); -router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), rbacVerify.handler(), dataIn); -router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), rbacVerify.handler(), dataOut); -router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), rbacVerify.handler(),DatasetCreate) -router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), rbacVerify.handler(), DatasetUpdate) -router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), rbacVerify.handler(), DatasetRead) -router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), rbacVerify.handler(), DatasetList) -router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), rbacVerify.handler(), dataExhaust); -router.post("/template/create", setDataToRequestObject("api.query.template.create"), rbacVerify.handler(), createQueryTemplate); -router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), rbacVerify.handler(), readQueryTemplate); -router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), rbacVerify.handler(), deleteQueryTemplate); -router.post("/template/list", setDataToRequestObject("api.query.template.list"), rbacVerify.handler(), listQueryTemplates); -router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), rbacVerify.handler(), updateQueryTemplate); -router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), rbacVerify.handler(), eventValidation); -router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), rbacVerify.handler(), queryTemplate); -router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), GenerateSignedURL); -router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), rbacVerify.handler(), DatasetStatusTansition); -router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), datasetHealth); -router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), datasetReset); -router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DataSchemaGenerator); -router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DatasetExport); -router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), rbacVerify.handler(), DatasetCopy); -router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), rbacVerify.handler(), ConnectorsList); -router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), rbacVerify.handler(), ConnectorsRead); -router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), rbacVerify.handler(), DatasetImport); +router.post("/data/in/:datasetId", setDataToRequestObject("api.data.in"), onRequest({ entity: Entity.Data_in }), telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), dataIn); +router.post("/data/query/:datasetId", setDataToRequestObject("api.data.out"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), dataOut); +router.post("/datasets/create", setDataToRequestObject("api.datasets.create"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.createDataset, operationType: OperationType.CREATE}), checkRBAC.handler(),DatasetCreate) +router.patch("/datasets/update", setDataToRequestObject("api.datasets.update"), onRequest({ entity: Entity.Management }),telemetryAuditStart({action: telemetryActions.updateDataset, operationType: OperationType.UPDATE}), checkRBAC.handler(), DatasetUpdate) +router.get("/datasets/read/:dataset_id", setDataToRequestObject("api.datasets.read"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readDataset, operationType: OperationType.GET}), checkRBAC.handler(), DatasetRead) +router.post("/datasets/list", setDataToRequestObject("api.datasets.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listDatasets, operationType: OperationType.LIST}), checkRBAC.handler(), DatasetList) +router.get("/data/exhaust/:datasetId", setDataToRequestObject("api.data.exhaust"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.datasetExhaust, operationType: OperationType.GET}), checkRBAC.handler(), dataExhaust); +router.post("/template/create", setDataToRequestObject("api.query.template.create"), checkRBAC.handler(), createQueryTemplate); +router.get("/template/read/:templateId", setDataToRequestObject("api.query.template.read"), checkRBAC.handler(), readQueryTemplate); +router.delete("/template/delete/:templateId", setDataToRequestObject("api.query.template.delete"), checkRBAC.handler(), deleteQueryTemplate); +router.post("/template/list", setDataToRequestObject("api.query.template.list"), checkRBAC.handler(), listQueryTemplates); +router.patch("/template/update/:templateId", setDataToRequestObject("api.query.template.update"), checkRBAC.handler(), updateQueryTemplate); +router.post("/schema/validate", setDataToRequestObject("api.schema.validator"), checkRBAC.handler(), eventValidation); +router.post("/template/query/:templateId", setDataToRequestObject("api.query.template.query"), checkRBAC.handler(), queryTemplate); +router.post("/files/generate-url", setDataToRequestObject("api.files.generate-url"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), GenerateSignedURL); +router.post("/datasets/status-transition", setDataToRequestObject("api.datasets.status-transition"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.createTransformation, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetStatusTansition); +router.post("/datasets/health", setDataToRequestObject("api.dataset.health"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetHealth); +router.post("/datasets/reset/:datasetId", setDataToRequestObject("api.dataset.reset"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), datasetReset); +router.post("/datasets/dataschema", setDataToRequestObject("api.datasets.dataschema"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DataSchemaGenerator); +router.get("/datasets/export/:dataset_id", setDataToRequestObject("api.datasets.export"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetExport); +router.post("/datasets/copy", setDataToRequestObject("api.datasets.copy"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.copyDataset, operationType: OperationType.CREATE}), checkRBAC.handler(), DatasetCopy); +router.post("/connectors/list", setDataToRequestObject("api.connectors.list"), onRequest({ entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.listConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsList); +router.get("/connectors/read/:id", setDataToRequestObject("api.connectors.read"), onRequest({entity: Entity.Management }), telemetryAuditStart({action: telemetryActions.readConnectors, operationType: OperationType.GET}), checkRBAC.handler(), ConnectorsRead); +router.post("/datasets/import", setDataToRequestObject("api.datasets.import"), onRequest({ entity: Entity.Management }), checkRBAC.handler(), DatasetImport); //Wrapper Service -router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), rbacVerify.handler(), sqlQuery); \ No newline at end of file +router.post("/obsrv/data/sql-query", setDataToRequestObject("api.obsrv.data.sql-query"), onRequest({ entity: Entity.Data_out }), checkRBAC.handler(), sqlQuery); \ No newline at end of file From 40049e93f4390b41a8752b973dd3d872aeb41190 Mon Sep 17 00:00:00 2001 From: yashashk Date: Tue, 3 Sep 2024 19:55:57 +0530 Subject: [PATCH 183/235] #OBS-I186 : Logic moved to separate function --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index e34a996b..d2a362d3 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -36,7 +36,7 @@ const datasetRead = async (req: Request, res: Response) => { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } if (dataset.connectors_config) { - dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); + dataset.connectors_config = processConnectorsConfig(dataset.connectors_config); } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } From 7fbcf742e214c3bd64d9cd92fefb838a1dbbf692 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Sep 2024 19:56:18 +0530 Subject: [PATCH 184/235] #OBS-I185 : fix: test case fixes --- .../DataIngestion/DataIngestionController.ts | 1 - .../controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition.ts | 2 +- .../DataIngestTest/DataIngestionTest.spec.ts | 61 ++++++++++++------- .../DataOutTest/DataQueryTest.spec.ts | 8 +-- .../DatasetRead/DatasetRead.spec.ts | 29 +++++++-- 6 files changed, 70 insertions(+), 33 deletions(-) diff --git a/api-service/src/controllers/DataIngestion/DataIngestionController.ts b/api-service/src/controllers/DataIngestion/DataIngestionController.ts index 69785067..46f829e9 100644 --- a/api-service/src/controllers/DataIngestion/DataIngestionController.ts +++ b/api-service/src/controllers/DataIngestion/DataIngestionController.ts @@ -23,7 +23,6 @@ const errorObject = { } } const apiId = "api.data.in"; -const errorCode = "DATASET_UPDATE_FAILURE" const dataIn = async (req: Request, res: Response) => { diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 5ae41826..025ef05f 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -78,7 +78,7 @@ const readDataset = async (datasetId: string, attributes: string[]): Promise) => { let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) defaultConfigs = _.omit(defaultConfigs, ["router_config"]) defaultConfigs = _.omit(defaultConfigs, "dedup_config.dedup_key"); - if (draftDataset?.type === 'master') { + if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue) => { diff --git a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts index 487cd77e..1ff1c9b4 100644 --- a/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataIngestTest/DataIngestionTest.spec.ts @@ -36,15 +36,13 @@ describe("DATA INGEST API", () => { it("it should ingest data for individual event", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: "local.test.topic", - }, - extraction_config: { - is_batch_event: false, - extraction_key: "events", - batch_id: "id" - } + dataset_config: { + entry_topic: "local.test.topic", + }, + extraction_config: { + is_batch_event: false, + extraction_key: "events", + batch_id: "id" } }) }) @@ -71,10 +69,8 @@ describe("DATA INGEST API", () => { it("it should ingest data successfully", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: "local.test.topic", - } + dataset_config: { + entry_topic: "local.test.topic", } }) }) @@ -97,13 +93,37 @@ describe("DATA INGEST API", () => { }) }); + it("it should ingest data successfully v2", (done) => { + chai.spy.on(Dataset, "findOne", () => { + return Promise.resolve({ + api_version: "v2", + entry_topic: "local.test.topic", + }) + }) + const connectionStub = sinon.stub(kafkaModule, "connect").returns(true); + const sendStub = sinon.stub(kafkaModule, "send").returns(resultResponse); + chai + .request(app) + .post(apiEndpoint) + .send(TestInputsForDataIngestion.SAMPLE_INPUT_1) + .end((err, res) => { + res.should.have.status(200); + res.body.should.be.a("object"); + res.body.should.have.property("result"); + res.body.id.should.be.eq("api.data.in"); + res.body.result.message.should.be.eq("Data ingested successfully") + connectionStub.restore() + sendStub.restore() + chai.spy.restore(Dataset, "findOne") + done() + }) + }); + it("Failed to connect kafka.", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: { - entry_topic: "local.test.topic", - } + dataset_config: { + entry_topic: "local.test.topic", } }) }) @@ -125,9 +145,7 @@ describe("DATA INGEST API", () => { it("Entry topic not found", (done) => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ - dataValues: { - dataset_config: {} - } + dataset_config: {} }) }) @@ -195,8 +213,7 @@ describe("DATA INGEST API", () => { res.body.should.be.a("object") res.body.id.should.be.eq("api.data.in"); res.body.params.status.should.be.eq("FAILED"); - res.body.error.code.should.be.eq("DATA_INGESTION_FAILED"); - res.body.error.message.should.be.eq("Failed to ingest data") + res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); }); diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 96d53e38..9f5cf5b3 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -89,8 +89,8 @@ describe("QUERY API TESTS", () => { res.body.params.status.should.be.eq("FAILED"); res.body.params.msgid.should.be.eq(msgid); res.body.params.should.have.property("resmsgid"); - res.body.error.message.should.be.eq("Unable to process the query."); - res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); + res.body.error.message.should.be.eq("Request failed with status code 500"); + res.body.error.code.should.be.eq("ERR_BAD_RESPONSE"); res.body.responseCode.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); @@ -115,8 +115,8 @@ describe("QUERY API TESTS", () => { res.body.params.status.should.be.eq("FAILED"); res.body.params.msgid.should.be.eq(msgid); res.body.params.should.have.property("resmsgid"); - res.body.error.message.should.be.eq("Unable to process the query."); - res.body.error.code.should.be.eq("INTERNAL_SERVER_ERROR"); + res.body.error.message.should.be.eq("Request failed with status code 500"); + res.body.error.code.should.be.eq("ERR_BAD_RESPONSE"); res.body.responseCode.should.be.eq("INTERNAL_SERVER_ERROR"); done(); }); diff --git a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts index cfd333eb..c3e766e7 100644 --- a/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetRead/DatasetRead.spec.ts @@ -31,9 +31,18 @@ describe("DATASET READ API", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "name": "sb-telemetry", "version": 1 }) }) + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(ConnectorInstances, "findAll", () => { + return Promise.resolve([]) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) chai .request(app) - .get("/v2/datasets/read/sb-telemetry?fields=name,version") + .get("/v2/datasets/read/sb-telemetry?fields=name,version,connectors_config,transformations_config") .end((err, res) => { res.should.have.status(httpStatus.OK); res.body.should.be.a("object") @@ -42,7 +51,7 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.name.should.be.eq("sb-telemetry") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ name: "sb-telemetry", version: 1 })) + result.should.be.eq(JSON.stringify({ "name": "sb-telemetry", "version": 1, "connectors_config": [], "transformations_config": [] })) done(); }); }); @@ -69,6 +78,15 @@ describe("DATASET READ API", () => { }); it("Dataset read success: Fetch live dataset when mode param not provided", (done) => { + chai.spy.on(DatasetTransformations, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) + }) + chai.spy.on(ConnectorInstances, "findAll", () => { + return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) + }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) chai.spy.on(Dataset, "findOne", () => { return Promise.resolve(TestInputsForDatasetRead.LIVE_SCHEMA) }) @@ -83,7 +101,7 @@ describe("DATASET READ API", () => { res.body.result.should.be.a("object") res.body.result.status.should.be.eq("Live") const result = JSON.stringify(res.body.result) - result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA })) + result.should.be.eq(JSON.stringify({ ...TestInputsForDatasetRead.LIVE_SCHEMA, connectors_config: TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2, transformations_config: TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA })) done(); }); }); @@ -98,6 +116,9 @@ describe("DATASET READ API", () => { chai.spy.on(DatasetTransformations, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.TRANSFORMATIONS_SCHEMA) }) + chai.spy.on(DatasetSourceConfig, "findAll", () => { + return Promise.resolve([]) + }) chai.spy.on(ConnectorInstances, "findAll", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V2) }) @@ -267,7 +288,7 @@ describe("DATASET READ API", () => { return Promise.resolve(TestInputsForDatasetRead.CONNECTORS_SCHEMA_V1) }) chai.spy.on(Dataset, "findAll", () => { - return Promise.resolve([{"dataset_id":"master_dataset", "dataset_config":{"cache_config":{"redis_db":20}}}]) + return Promise.resolve([{ "dataset_id": "master_dataset", "dataset_config": { "cache_config": { "redis_db": 20 } } }]) }) chai .request(app) From 7e330d384828f445b6f671b1e935ebbf8852b4e0 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 3 Sep 2024 20:02:55 +0530 Subject: [PATCH 185/235] #OBS-I185 : fix: linting fix --- .../DatasetMetrics/DatasetMetricsController.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts index 0e0d2193..903d393c 100644 --- a/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts +++ b/api-service/src/controllers/DatasetMetrics/DatasetMetricsController.ts @@ -10,7 +10,7 @@ import axios from "axios"; import { config } from "../../configs/Config"; const getBaseUrl = (url: string) => { - if (_.startsWith(url, '/prom')) return config.query_api.prometheus.url + _.replace(url, '/prom', '') + if (_.startsWith(url, "/prom")) return config.query_api.prometheus.url + _.replace(url, "/prom", "") } const datasetMetrics = async (req: Request, res: Response) => { @@ -19,13 +19,13 @@ const datasetMetrics = async (req: Request, res: Response) => { logger.error({ message: isValidSchema?.message, code: "INVALID_QUERY" }) throw obsrvError("", "INVALID_QUERY", isValidSchema.message, "BAD_REQUEST", 400) } - let { query } = req.body || {}; - let endpoint = query.url; - if (_.startsWith(endpoint, '/prom')) { + const { query } = req.body || {}; + const endpoint = query.url; + if (_.startsWith(endpoint, "/prom")) { query.url = getBaseUrl(endpoint) const { url, method, headers = {}, body = {}, params = {}, ...rest } = query; const apiResponse = await axios.request({ url, method, headers, params, data: body, ...rest }) - const data = _.get(apiResponse, 'data'); + const data = _.get(apiResponse, "data"); return res.json(data); } else { From 7dff8607580e16d23e3736814fab259a9afa355a Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 20:03:12 +0530 Subject: [PATCH 186/235] #OBS-I164: modified config and rbac middleware --- api-service/src/configs/Config.ts | 2 +- api-service/src/middlewares/RBAC_middleware.ts | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/api-service/src/configs/Config.ts b/api-service/src/configs/Config.ts index 81702432..b26a6906 100644 --- a/api-service/src/configs/Config.ts +++ b/api-service/src/configs/Config.ts @@ -114,5 +114,5 @@ export const config = { "access_token": process.env.grafana_token || "" }, "user_token_public_key": process.env.user_token_public_key || "", - "is_RBAC_enabled": false + "is_RBAC_enabled": process.env.is_rbac_enabled || "false", } diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index b84b2edb..d36babf0 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -2,6 +2,7 @@ import { Request, Response, NextFunction } from "express"; import jwt from "jsonwebtoken"; import { ResponseHandler } from "../helpers/ResponseHandler"; import { config } from "../configs/Config"; +import _ from "lodash"; enum roles { Admin = "admin", @@ -122,7 +123,9 @@ export default { name: "rbac:middleware", handler: () => (req: Request, res: Response, next: NextFunction) => { try { - if (config.is_RBAC_enabled === true) { + if (_.lowerCase(config.is_RBAC_enabled) === "false") { + next(); + } else { const public_key = config.user_token_public_key; const token = req.get("x-user-token"); if (!token) { @@ -148,9 +151,9 @@ export default { res ); } - if (decoded && typeof decoded == "object") { + if (decoded && _.isObject(decoded)) { const action = (req as any).id; - const hasAccess = decoded.roles.some( + const hasAccess = decoded?.roles?.some( (role: string) => accessControl[role] && accessControl[role].includes(action) ); @@ -168,8 +171,6 @@ export default { next(); } }); - } else { - next(); } } catch (error) { next(error); From b83a8082f7b234e68667526d35d47a1a31c33d9e Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 3 Sep 2024 20:15:28 +0530 Subject: [PATCH 187/235] #OBS-I164: added jsonwebtoken package --- api-service/package.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api-service/package.json b/api-service/package.json index 1d49c029..acde61cc 100644 --- a/api-service/package.json +++ b/api-service/package.json @@ -33,6 +33,7 @@ "express": "^5.0.0-beta.3", "http-errors": "^2.0.0", "http-status": "^1.5.3", + "jsonwebtoken": "^9.0.1", "kafka-node": "^5.0.0", "kafkajs": "^2.2.4", "kafkajs-snappy": "^1.1.0", @@ -65,6 +66,7 @@ "@types/compression": "^1.7.2", "@types/express": "^4.17.14", "@types/http-errors": "^2.0.1", + "@types/jsonwebtoken": "^9.0.6", "@types/kafkajs": "^1.9.0", "@types/knex": "^0.16.1", "@types/lodash": "^4.14.190", From 39b14e317d2ddaeb9d6598399c5e0f58f168b8bc Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:01:47 +0530 Subject: [PATCH 188/235] #OBS-I165: added userInfo from token to request object --- api-service/src/middlewares/RBAC_middleware.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index d36babf0..22bd40b3 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -152,6 +152,7 @@ export default { ); } if (decoded && _.isObject(decoded)) { + (req as any).userInfo = decoded; const action = (req as any).id; const hasAccess = decoded?.roles?.some( (role: string) => From b85cabfc4442186898b158af9b15b31a9d481191 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:03:41 +0530 Subject: [PATCH 189/235] #OBS-I165: updated telemetry to use user role --- api-service/src/services/telemetry.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index d4eca61b..f24995a9 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -10,15 +10,15 @@ const telemetryTopic = _.get(appConfig, "telemetry_dataset"); export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } -const getDefaults = () => { +const getDefaults = (userInfo:any) => { return { eid: "AUDIT", ets: Date.now(), ver: "1.0.0", mid: v4(), actor: { - id: "SYSTEM", - type: "User" + id: userInfo?.id || "SYSTEM", + type: userInfo?.roles[0] || "User", }, context: { env, @@ -128,7 +128,7 @@ export const processAuditEvents = (request: Request) => { _.set(auditEvent, "edata.transition.toState", toState); _.set(auditEvent, "edata.transition.fromState", fromState); } - const telemetryEvent = getDefaults(); + const telemetryEvent = getDefaults((request as any)?.userInfo); _.set(telemetryEvent, "edata", edata); _.set(telemetryEvent, "object", { ...(object.id && object.type && { ...object, ver: "1.0.0" }) }); sendTelemetryEvents(telemetryEvent); From 627a60bfed3554ca6b5478b7a4f9a6b98b3cfd9d Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:08:18 +0530 Subject: [PATCH 190/235] #OBS-I165: updated datasetCreate api to add userRole as created_by --- api-service/src/controllers/DatasetCreate/DatasetCreate.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 2867457b..2e476b7c 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -34,6 +34,8 @@ const datasetCreate = async (req: Request, res: Response) => { await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) + const userRole = (req as any)?.userInfo?.roles[0] || "SYSTEM"; + _.set(draftDataset, "created_by", userRole); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } From e33fb97f6c4493c6534598a4660527cf7f10fe74 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:12:00 +0530 Subject: [PATCH 191/235] #OBS-I165: added userRole when migrating and create table from live --- api-service/src/controllers/DatasetRead/DatasetRead.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index e1853e57..5e3a869c 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -30,8 +30,9 @@ const datasetRead = async (req: Request, res: Response) => { validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; + const userRole = (req as any)?.userInfo?.roles[0]; const attributes = !fields ? defaultFields : _.split(fields, ","); - const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes) : await readDataset(dataset_id, attributes) + const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userRole) : await readDataset(dataset_id, attributes) if (!dataset) { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } @@ -41,19 +42,19 @@ const datasetRead = async (req: Request, res: Response) => { ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } -const readDraftDataset = async (datasetId: string, attributes: string[]): Promise => { +const readDraftDataset = async (datasetId: string, attributes: string[], userRole: string): Promise => { const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); if (draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); - const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset) + const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset, userRole) return _.pick(dataset, attributes); } const liveDataset = await datasetService.getDataset(datasetId, undefined, true); if (liveDataset) { - const dataset = await datasetService.createDraftDatasetFromLive(liveDataset) + const dataset = await datasetService.createDraftDatasetFromLive(liveDataset, userRole) return _.pick(dataset, attributes); } From 27c0940d29b40c5276afd4a5c9ab68db02764097 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:14:04 +0530 Subject: [PATCH 192/235] #OBS-I165: added userRole for data copy api --- api-service/src/controllers/DatasetCopy/DatasetCopy.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 9308dd71..8ddc9fe2 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -40,6 +40,8 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); + const userRole = (req as any)?.userInfo?.roles[0]; + _.set(dataset, "created_by", userRole); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { if (err?.name === "SequelizeUniqueConstraintError") { From dde0858731df03451c996e42fa30d1e71e0bb963 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:15:04 +0530 Subject: [PATCH 193/235] #OBS-I165: added userRole for dataset Update --- api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 274ad3c4..679b3fcc 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -59,6 +59,7 @@ const datasetUpdate = async (req: Request, res: Response) => { validateDataset(datasetModel, req) const draftDataset = mergeDraftDataset(datasetModel, datasetReq); + _.set(draftDataset, "updated_by", (req as any)?.userInfo?.roles[0] ) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } From 8e946954b07d5f61c3c2a8deeb7cf882033b00a4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:15:49 +0530 Subject: [PATCH 194/235] #OBS-I165: added userRole for dataset Import --- api-service/src/controllers/DatasetImport/DatasetImport.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index e390d8bd..97314ecc 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -15,18 +15,21 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } + const userRole = (req as any)?.userInfo?.roles[0]; + _.set(datasetPayload, "created_by", userRole); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) - const dataset = await importDataset(updatedDataset, overwrite); + const dataset = await importDataset(updatedDataset, overwrite, userRole); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); } -const importDataset = async (dataset: Record, overwrite: string | any) => { +const importDataset = async (dataset: Record, overwrite: string | any, userRole : string) => { const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) if (response?.name === "SequelizeUniqueConstraintError") { if (overwrite === "true") { + _.set(dataset, "updated_by", userRole); const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{ throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) From 7556358af993323d5ed9d68f0e958a0be471661b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:16:03 +0530 Subject: [PATCH 195/235] #OBS-I165: added userRole for dataset status transition --- .../DatasetStatusTransition.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index c2a18021..eb03c7b5 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -55,6 +55,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const userRole = (req as any)?.userInfo?.roles[0] || "SYSTEM"; validateDataset(dataset, dataset_id, status); switch (status) { @@ -62,13 +63,13 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await deleteDataset(dataset); break; case "ReadyToPublish": - await readyForPublish(dataset); + await readyForPublish(dataset, userRole); break; case "Live": - await publishDataset(dataset); + await publishDataset(dataset, userRole); break; case "Retire": - await retireDataset(dataset); + await retireDataset(dataset, userRole); break; case "Archive": await archiveDataset(dataset); @@ -90,7 +91,7 @@ const deleteDataset = async (dataset: Record) => { } -const readyForPublish = async (dataset: Record) => { +const readyForPublish = async (dataset: Record, updated_by: any) => { const draftDataset: any = await datasetService.getDraftDataset(dataset.dataset_id) let defaultConfigs: any = _.cloneDeep(defaultDatasetConfig) @@ -99,7 +100,14 @@ const readyForPublish = async (dataset: Record) => { if (draftDataset?.type === "master") { defaultConfigs = _.omit(defaultConfigs, "dataset_config.keys_config.data_key"); } - _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue) => { + _.set(draftDataset, "updated_by", updated_by); + _.mergeWith(draftDataset, defaultConfigs, draftDataset, (objValue, srcValue ,key) => { + if (key === "created_by"|| key === "updated_by") { + if (objValue !== "SYSTEM") { + return objValue; + } + return srcValue; + } if (_.isBoolean(objValue) && _.isBoolean(srcValue)) { return objValue; } @@ -126,10 +134,11 @@ const readyForPublish = async (dataset: Record) => { * * @param dataset */ -const publishDataset = async (dataset: Record) => { +const publishDataset = async (dataset: Record, userRole: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record - + _.set(draftDataset, ["updated_by"], userRole); + console.log(draftDataset); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) await datasetService.publishDataset(draftDataset) @@ -208,10 +217,10 @@ const updateMasterDataConfig = async (draftDataset: Record) => { } } -const retireDataset = async (dataset: Record) => { +const retireDataset = async (dataset: Record, updated_by: any) => { await canRetireIfMasterDataset(dataset); - await datasetService.retireDataset(dataset); + await datasetService.retireDataset(dataset, updated_by); await restartPipeline(dataset); } From 40987b3268a19254a9d6e75f4f66d6f5e762d2a4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 13:17:05 +0530 Subject: [PATCH 196/235] #OBS-I165: modified Dataset Service to update the userRoles --- api-service/src/services/DatasetService.ts | 25 ++++++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 2e800212..4bb27ed7 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -92,9 +92,10 @@ class DatasetService { return responseData; } - migrateDraftDataset = async (datasetId: string, dataset: Record): Promise => { + migrateDraftDataset = async (datasetId: string, dataset: Record, userRole: string): Promise => { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); + _.set(draftDataset, "updated_by", userRole); const transaction = await sequelize.transaction(); try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); @@ -166,7 +167,7 @@ class DatasetService { } } - createDraftDatasetFromLive = async (dataset: Model) => { + createDraftDatasetFromLive = async (dataset: Model, userRole: string) => { const draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); const dataset_config: any = _.get(dataset, "dataset_config"); @@ -232,6 +233,7 @@ class DatasetService { draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft + draftDataset["created_by"] = userRole; const result = await DatasetDraft.create(draftDataset); return _.get(result, "dataValues") } @@ -256,14 +258,14 @@ class DatasetService { } } - retireDataset = async (dataset: Record) => { + retireDataset = async (dataset: Record, updatedBy: any) => { const transaction = await sequelize.transaction(); try { - await Dataset.update({ status: DatasetStatus.Retired }, { where: { id: dataset.id }, transaction }); - await DatasetSourceConfig.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await Datasource.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); - await DatasetTransformations.update({ status: DatasetStatus.Retired }, { where: { dataset_id: dataset.id }, transaction }); + await Dataset.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { id: dataset.id }, transaction }); + await DatasetSourceConfig.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); + await Datasource.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); + await DatasetTransformations.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); await this.deleteDruidSupervisors(dataset); } catch (err: any) { @@ -321,30 +323,39 @@ class DatasetService { private createDruidDataSource = async (draftDataset: Record, transaction: Transaction) => { + const {created_by, updated_by} = draftDataset; const allFields = await tableGenerator.getAllFields(draftDataset, "druid"); const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); await DatasourceDraft.create(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + const {created_by, updated_by} = draftDataset; const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); await DatasourceDraft.create(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { + const {created_by, updated_by} = draftDataset; const allFields = await tableGenerator.getAllFields(draftDataset, "hudi"); const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const dsId = _.join([draftDataset.dataset_id, "events", "hudi"], "_") const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) + _.set(draftDatasource, "created_by", created_by); + _.set(draftDatasource, "updated_by", updated_by); await DatasourceDraft.create(draftDatasource, { transaction }) } From e013c29bf13289afeef9861495b743f54d9f6243 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 18:05:17 +0530 Subject: [PATCH 197/235] #OBS-I165: added permission for queryTemplateUpdate api --- api-service/src/middlewares/RBAC_middleware.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 22bd40b3..92d352c8 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -70,6 +70,7 @@ const accessControl: AccessControl = { permissions.QueryTemplateCreate, permissions.QueryTemplateRead, permissions.QueryTemplateDelete, + permissions.QueryTemplateUpdate, permissions.GenerateSignedUrl, permissions.SchemaValidator, permissions.DatasetSchemaGenerator, @@ -89,6 +90,7 @@ const accessControl: AccessControl = { permissions.QueryTemplateCreate, permissions.QueryTemplateRead, permissions.QueryTemplateDelete, + permissions.QueryTemplateUpdate, permissions.GenerateSignedUrl, permissions.SchemaValidator, permissions.DatasetSchemaGenerator, @@ -111,6 +113,7 @@ const accessControl: AccessControl = { permissions.QueryTemplateCreate, permissions.QueryTemplateRead, permissions.QueryTemplateDelete, + permissions.QueryTemplateUpdate, permissions.GenerateSignedUrl, permissions.SchemaValidator, permissions.DatasetSchemaGenerator, From a88f30cdfd05e17b2d40dd0ee776c913e0971e58 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 10 Sep 2024 18:10:33 +0530 Subject: [PATCH 198/235] #OBS-I165: added userRole for query template create and update --- .../controllers/CreateQueryTemplate/CreateTemplateController.ts | 1 + .../controllers/UpdateQueryTemplate/UpdateTemplateController.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index 90808e7d..94b4cffd 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -42,6 +42,7 @@ export const createQueryTemplate = async (req: Request, res: Response) => { } const data = transformRequest(requestBody, templateName); + _.set(data, "created_by", (req as any)?.userInfo?.roles[0]); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) return ResponseHandler.successResponse(req, res, { status: 200, data: { template_id: templateId, template_name: templateName, message: `The query template has been saved successfully` } }); diff --git a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts index 114d370e..30f3c5c5 100644 --- a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts +++ b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts @@ -38,7 +38,7 @@ export const updateQueryTemplate = async (req: Request, res: Response) => { logger.error({ apiId, msgid, resmsgid, templateId, requestBody: req?.body, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, code: "QUERY_TEMPLATE_INVALID_INPUT" }) return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, errCode: "BAD_REQUEST", code: "QUERY_TEMPLATE_INVALID_INPUT" }, req, res) } - + requestBody.request.updated_by = (req as any)?.userInfo?.roles[0]; await QueryTemplate.update(requestBody?.request, { where: { template_id: templateId } }) logger.info({ apiId, msgid, resmsgid, templateId, requestBody, message: `Query template updated successfully` }) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Query template updated successfully", templateId } }); From c48efd77113ca6eb0a83243549ae5815c057c388 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 08:39:27 +0530 Subject: [PATCH 199/235] #OBS-I165: added userRole for alerts api --- api-service/src/controllers/Alerts/Alerts.ts | 7 ++++++- api-service/src/services/managers/index.ts | 12 ++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 278ac586..131fc9a2 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -13,6 +13,7 @@ const telemetryObject = { type: "alert", ver: "1.0.0" }; const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const alertPayload = getAlertPayload(req.body); + _.set(alertPayload, "created_by", (req as any)?.userInfo?.roles[0]); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); @@ -30,6 +31,7 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const { alertId } = req.params; const rulePayload: Record | null = await getAlertRule(alertId); if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); if (rulePayload.status == "live") { await deleteAlertRule(rulePayload, false); } @@ -87,6 +89,7 @@ const deleteAlertHandler = async (req: Request, res: Response, next: NextFunctio return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); } const rulePayload = ruleModel.toJSON(); + _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); await deleteAlertRule(rulePayload, hardDelete === "true"); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); @@ -104,11 +107,13 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } const rulePayload = ruleModel.toJSON(); if (rulePayload.status == "live") { + _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); await deleteAlertRule(rulePayload, false); await retireAlertSilence(alertId); } const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); - await Alert.update({ ...updatedPayload, status: "draft" }, { where: { id: alertId } }); + const userRole = (req as any)?.userInfo?.roles[0]; + await Alert.update({ ...updatedPayload, status: "draft", updated_by: userRole }, { where: { id: alertId } }); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); } catch (error: any) { diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index 84b666d9..e6cf822f 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -24,16 +24,16 @@ const getService = (manager: string) => { }; export const publishAlert = async (payload: Record) => { - const { id, manager } = payload; + const { id, manager, updated_by } = payload; const service = getService(manager); const publishResponse = await service.publishAlert(payload) - await updateStatus(id, "live"); + await updateStatus(id, "live", updated_by); return publishResponse; }; -const updateStatus = (id: string, status: string) => { - return Alert.update({ status }, { where: { id } }); +const updateStatus = (id: string, status: string, updated_by: string) => { + return Alert.update({ status, updated_by }, { where: { id } }); } const deleteRule = (id: string) => { @@ -41,7 +41,7 @@ const deleteRule = (id: string) => { } export const deleteAlertRule = async (payload: Record, hardDelete: boolean) => { - const { id, manager, status } = payload; + const { id, manager, status, updated_by } = payload; if (status == "live") { try { @@ -56,7 +56,7 @@ export const deleteAlertRule = async (payload: Record, hardDelete: return deleteRule(id); } - return updateStatus(id, "retired"); + return updateStatus(id, "retired", updated_by); } From 5e49317f898bad1fc93d89456a35b76ce90469c6 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 08:39:46 +0530 Subject: [PATCH 200/235] #OBS-I165: added userRole for notifications api --- .../controllers/NotificationChannel/Notification.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index f6db4fd4..949c8670 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,6 +12,8 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; + const userRole = (request as any)?.userInfo?.role[0]; + _.set(payload, "created_by", userRole); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) @@ -32,6 +34,8 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } + const userRole = (request as any)?.userInfo?.role[0]; + _.set(updatedPayload, "updated_by", userRole); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -74,7 +78,8 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - await Notification.update({ status: "retired" }, { where: { id } }) + const userRole = (request as any)?.userInfo?.role[0]; + await Notification.update({ status: "retired", updated_by: userRole }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) @@ -91,7 +96,8 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - Notification.update({ status: "live" }, { where: { id } }); + const userRole = (request as any)?.userInfo?.role[0]; + Notification.update({ status: "live", updated_by: userRole }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) From 428db1b1ca825010a3ef7a5128a585233dfbeb57 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 08:40:04 +0530 Subject: [PATCH 201/235] #OBS-I165: added userRole for silences api --- api-service/src/controllers/Alerts/Silence.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index 32e2c531..fd2bee31 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -20,12 +20,14 @@ const createHandler = async (request: Request, response: Response, next: NextFun const start_date = new Date(startDate); const end_date = new Date(endDate); + const userRole = (request as any)?.userInfo?.roles[0]; const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, alert_id: alertId, start_time: start_date, end_time: end_date, + created_by : userRole, } const sileneResponse = await Silence.create(silenceBody); updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); @@ -78,10 +80,12 @@ const updateHandler = async (request: Request, response: Response, next: NextFun await updateSilence(silenceObject, payload); const updatedStartTime = new Date(payload.startTime); const updatedEndTime = new Date(payload.endTime); + const userRole = (request as any)?.userInfo?.roles[0]; const updatedSilence = { ...silenceObject, start_time: updatedStartTime, - end_time: updatedEndTime + end_time: updatedEndTime, + updated_by: userRole, } const silenceResponse = await Silence.update(updatedSilence, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } }) From d2bffb6b01061fe765365aa43e3b8ae2faefa0ff Mon Sep 17 00:00:00 2001 From: yashashk Date: Wed, 11 Sep 2024 12:48:03 +0530 Subject: [PATCH 202/235] #OBS-I86 : commented routes have regex --- api-service/src/app.ts | 2 +- api-service/src/routes/DruidProxyRouter.ts | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 962331e9..27b9fce6 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -22,7 +22,7 @@ app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); app.use("/", metricRouter); -app.use("*", ResponseHandler.routeNotFound); +// app.use("*", ResponseHandler.routeNotFound); app.use(obsrvErrorHandler); app.listen(config.api_port, () => { diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts index ca1aec91..70fa3ee6 100644 --- a/api-service/src/routes/DruidProxyRouter.ts +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -9,15 +9,15 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; export const druidProxyRouter = express.Router(); // Send a 410 Gone response to all V1 API calls -druidProxyRouter.all("/datasets/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/dataset/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/datasources/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/data/v1/*", ResponseHandler.goneResponse) -druidProxyRouter.all("/template/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/datasets/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/dataset/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/datasources/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/data/v1/*", ResponseHandler.goneResponse) +// druidProxyRouter.all("/template/v1/*", ResponseHandler.goneResponse) -// Druid Proxy APIs for Metabase integration -druidProxyRouter.post(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative) -druidProxyRouter.get(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet) -druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) -druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) -druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file +// // Druid Proxy APIs for Metabase integration +// druidProxyRouter.post(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative) +// druidProxyRouter.get(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet) +// druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) +// druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) +// druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file From 23abcc40839c947b195253a920b03b1797677d85 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 11 Sep 2024 16:41:04 +0530 Subject: [PATCH 203/235] #OBS-I77 : query api changes to check datasource avialability v2 --- .../src/controllers/DataOut/QueryValidator.ts | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index e569bdb3..9058603e 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -4,10 +4,11 @@ import * as _ from "lodash"; import moment from "moment"; import { getDatasourceList } from "../../services/DatasourceService"; import logger from "../../logger"; -import { getDatasourceListFromDruid } from "../../connections/druidConnection"; +import { druidHttpService, getDatasourceListFromDruid } from "../../connections/druidConnection"; import { apiId } from "./DataOutController"; import { ErrorObject } from "../../types/ResponseModel"; import { Parser } from "node-sql-parser"; +import { obsrvError } from "../../types/ObsrvError"; const parser = new Parser(); const momentFormat = "YYYY-MM-DD HH:MM:SS"; @@ -155,23 +156,35 @@ const validateQueryRules = (queryPayload: any, limits: any) => { : { message: "Invalid date range! the date range cannot be a null value", statusCode: 400, errCode: "BAD_REQUEST", code: errCode.invalidDateRange }; }; -const getDataSourceRef = async (datasetId: string, granularity?: string) => { +const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { const dataSources = await getDatasourceList(datasetId) if (_.isEmpty(dataSources)) { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Datasource ${datasetId} not available in datasource live table`, code: errCode.notFound }) throw { message: `Datasource ${datasetId} not available for querying`, statusCode: 404, errCode: "NOT_FOUND", code: errCode.notFound } as ErrorObject; } - const record = dataSources.filter((record: any) => { - const aggregatedRecord = _.get(record, "dataValues.metadata.aggregated") - if (granularity) - return aggregatedRecord && _.get(record, "dataValues.metadata.granularity") === granularity; + const record = dataSources.find((record: any) => { + const metadata = _.get(record, "dataValues.metadata", {}); + const { aggregated, granularity } = metadata; + if (!aggregated) { + return true; + } + return aggregated && srcGranularity ? granularity === srcGranularity : false; }); - return record[0]?.dataValues?.datasource_ref + return _.get(record, ["dataValues", "datasource_ref"]) +} + +const checkSupervisorAvailability = async (datasourceRef: string) => { + const { data } = await druidHttpService.get('/druid/coordinator/v1/loadstatus'); + const datasourceLoad = _.get(data, datasourceRef) + if (!(datasourceLoad && datasourceLoad === 100)) { + throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not fully available to query", "RANGE_NOT_SATISFIABLE", 416) + } } const setDatasourceRef = async (datasetId: string, payload: any): Promise => { const granularity = _.get(payload, "context.aggregationLevel") const datasourceRef = await getDataSourceRef(datasetId, granularity); + await checkSupervisorAvailability(datasourceRef) const existingDatasources = await getDatasourceListFromDruid(); if (!_.includes(existingDatasources.data, datasourceRef)) { From 6d83507003dfc6b867f079635cf4d6c59380195c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 11 Sep 2024 17:10:12 +0530 Subject: [PATCH 204/235] #OBS-I77 : fix: test cases --- .../src/controllers/DataOut/QueryValidator.ts | 2 +- .../DataOutTest/DataQueryTest.spec.ts | 36 +++++++++++++++++++ .../TemplateQuerying/TemplateQuerying.spec.ts | 11 ++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 9058603e..9d79069d 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -174,7 +174,7 @@ const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { } const checkSupervisorAvailability = async (datasourceRef: string) => { - const { data } = await druidHttpService.get('/druid/coordinator/v1/loadstatus'); + const { data } = await druidHttpService.get("/druid/coordinator/v1/loadstatus"); const datasourceLoad = _.get(data, datasourceRef) if (!(datasourceLoad && datasourceLoad === 100)) { throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not fully available to query", "RANGE_NOT_SATISFIABLE", 416) diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 9f5cf5b3..b7af9bbb 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -7,6 +7,7 @@ import { config } from "../../../configs/Config"; import chaiSpies from "chai-spies" import { describe, it } from "mocha"; import { Datasource } from "../../../models/Datasource"; +import { druidHttpService } from "../../../connections/druidConnection"; chai.use(chaiSpies) chai.should(); chai.use(chaiHttp); @@ -33,6 +34,11 @@ describe("QUERY API TESTS", () => { response ) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["telemetry-events.1_rollup"]) @@ -74,6 +80,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -100,6 +111,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -126,6 +142,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -153,6 +174,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -180,6 +206,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) @@ -223,6 +254,11 @@ describe("QUERY API TESTS", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_week"]) diff --git a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts index 8da1818b..55368903 100644 --- a/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts +++ b/api-service/src/tests/QueryTemplates/TemplateQuerying/TemplateQuerying.spec.ts @@ -8,6 +8,7 @@ import { Datasource } from "../../../models/Datasource"; import nock from "nock"; import { config } from "../../../configs/Config"; import { templateQueryApiFixtures } from "./Fixtures"; +import { druidHttpService } from "../../../connections/druidConnection"; const apiId = "api.query.template.query"; const msgid = "4a7f14c3-d61e-4d4f-be78-181834eeff6d" @@ -50,6 +51,11 @@ describe("QUERY TEMPLATE API", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_month": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_month"]) @@ -93,6 +99,11 @@ describe("QUERY TEMPLATE API", () => { return Promise.resolve(response) }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_month": 100 } + }) + }) nock(druidHost + ":" + druidPort) .get(listDruidDatasources) .reply(200, ["test.1_rollup_month"]) From e76e473b735580b48f4629a416b03196554f4944 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 17:22:38 +0530 Subject: [PATCH 205/235] #OBS-I165: corrections userRole access in notification --- .../src/controllers/NotificationChannel/Notification.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index 949c8670..bc470c36 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,7 +12,7 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; - const userRole = (request as any)?.userInfo?.role[0]; + const userRole = (request as any)?.userInfo?.roles[0]; _.set(payload, "created_by", userRole); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); @@ -34,7 +34,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } - const userRole = (request as any)?.userInfo?.role[0]; + const userRole = (request as any)?.userInfo?.roles[0]; _.set(updatedPayload, "updated_by", userRole); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); @@ -78,7 +78,7 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - const userRole = (request as any)?.userInfo?.role[0]; + const userRole = (request as any)?.userInfo?.roles[0]; await Notification.update({ status: "retired", updated_by: userRole }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -96,7 +96,7 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - const userRole = (request as any)?.userInfo?.role[0]; + const userRole = (request as any)?.userInfo?.roles[0]; Notification.update({ status: "live", updated_by: userRole }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { From 9096b75cc602afe0e7434a6f9fe1cc203721d54b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 18:20:38 +0530 Subject: [PATCH 206/235] #OBS-I165: added operations_admin role and updated permissions --- .../src/middlewares/RBAC_middleware.ts | 82 +++++++++++++++++++ 1 file changed, 82 insertions(+) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 92d352c8..e37bd916 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -5,6 +5,7 @@ import { config } from "../configs/Config"; import _ from "lodash"; enum roles { + OperationsAdmin = "operations_admin", Admin = "admin", DatasetManager = "dataset_manager", Viewer = "viewer", @@ -13,6 +14,28 @@ enum roles { } enum permissions { + AlertCreate = "api.alert.create", + AlertPublish = "api.alert.publish", + AlertUpdate = "api.alert.update", + AlertList = "api.alert.list", + AlertDelete = "api.alert.delete", + AlertDetails = "api.alert.getAlertDetails", + MetricCreate = "api.metric.add", + MetricList = "api.metric.list", + MetricUpdate = "api.metric.update", + MetricRemove = "api.metric.remove", + SilenceCreate = "api.alert.silence.create", + SilenceList = "api.alert.silence.list", + SilenceRead = "api.alert.silence.get", + SilenceEdit = "api.alert.silence.edit", + SilenceDelete = "api.alert.silence.delete", + NotificationChannelCreate = "api.alert.notification.create", + NotificationChannelList = "api.alert.notification.list", + NotificationChannelPublish = "api.alert.notification.publish", + NotificationChannelTest = "api.alert.notification.test", + NotificationChannelUpdate = "api.alert.notification.update", + NotificationChannelRetire = "api.alert.notification.retire", + NotificationChannelRead = "api.alert.notification.get", DatasetCreate = "api.datasets.create", DatasetUpdate = "api.datasets.update", DatasetRead = "api.datasets.read", @@ -54,6 +77,13 @@ const accessControl: AccessControl = { permissions.SQLQuery, permissions.DataOut, permissions.DataExhaust, + permissions.AlertList, + permissions.AlertDetails, + permissions.MetricList, + permissions.SilenceList, + permissions.SilenceRead, + permissions.NotificationChannelList, + permissions.NotificationChannelRead, ], [roles.DatasetCreator]: [ permissions.DatasetList, @@ -74,6 +104,13 @@ const accessControl: AccessControl = { permissions.GenerateSignedUrl, permissions.SchemaValidator, permissions.DatasetSchemaGenerator, + permissions.AlertList, + permissions.AlertDetails, + permissions.MetricList, + permissions.SilenceList, + permissions.SilenceRead, + permissions.NotificationChannelList, + permissions.NotificationChannelRead, ], [roles.DatasetManager]: [ permissions.DatasetList, @@ -96,6 +133,13 @@ const accessControl: AccessControl = { permissions.DatasetSchemaGenerator, permissions.DatasetReset, permissions.DatasetStatusTransition, + permissions.AlertList, + permissions.AlertDetails, + permissions.MetricList, + permissions.SilenceList, + permissions.SilenceRead, + permissions.NotificationChannelList, + permissions.NotificationChannelRead, ], [roles.Admin]: [ permissions.DatasetCreate, @@ -119,6 +163,44 @@ const accessControl: AccessControl = { permissions.DatasetSchemaGenerator, permissions.DatasetReset, permissions.DatasetStatusTransition, + permissions.AlertList, + permissions.AlertDetails, + permissions.MetricList, + permissions.SilenceList, + permissions.SilenceRead, + permissions.NotificationChannelList, + permissions.NotificationChannelRead, + ], + [roles.OperationsAdmin]: [ + permissions.AlertCreate, + permissions.AlertPublish, + permissions.AlertUpdate, + permissions.AlertList, + permissions.AlertDelete, + permissions.AlertDetails, + permissions.MetricCreate, + permissions.MetricList, + permissions.MetricUpdate, + permissions.MetricRemove, + permissions.SilenceCreate, + permissions.SilenceList, + permissions.SilenceRead, + permissions.SilenceEdit, + permissions.SilenceDelete, + permissions.NotificationChannelCreate, + permissions.NotificationChannelList, + permissions.NotificationChannelPublish, + permissions.NotificationChannelTest, + permissions.NotificationChannelUpdate, + permissions.NotificationChannelRetire, + permissions.NotificationChannelRead, + permissions.DatasetList, + permissions.DatasetRead, + permissions.DatasetExport, + permissions.ConnectorRead, + permissions.SQLQuery, + permissions.DataOut, + permissions.DataExhaust, ], }; From 45adebe6f5af5afc913905f3a431991cdecf497b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Wed, 11 Sep 2024 18:21:15 +0530 Subject: [PATCH 207/235] #OBS-I165: added rbac middleware for alert routers --- api-service/src/routes/AlertsRouter.ts | 49 +++++++++++++------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/api-service/src/routes/AlertsRouter.ts b/api-service/src/routes/AlertsRouter.ts index 01c843dd..6044e9fe 100644 --- a/api-service/src/routes/AlertsRouter.ts +++ b/api-service/src/routes/AlertsRouter.ts @@ -4,38 +4,39 @@ import { setDataToRequestObject } from "../middlewares/setDataToRequestObject"; import customAlertHandler from "../controllers/Alerts/Alerts"; import metricAliasHandler from "../controllers/Alerts/Metric"; import silenceHandler from "../controllers/Alerts/Silence"; +import checkRBAC from "../middlewares/RBAC_middleware"; export const alertsRouter = express.Router(); // Notifications -alertsRouter.post("/notifications/search", setDataToRequestObject("api.alert.notification.list"), notificationHandler.listHandler); -alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), notificationHandler.createHandler); -alertsRouter.get("/notifications/publish/:id", setDataToRequestObject("api.alert.notification.publish"), notificationHandler.publishHandler); -alertsRouter.post("/notifications/test", setDataToRequestObject("api.alert.notification.test"), notificationHandler.testNotifationChannelHandler); -alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), notificationHandler.updateHandler); -alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), notificationHandler.retireHandler); -alertsRouter.get("/notifications/get/:id", setDataToRequestObject("api.alert.notification.get"), notificationHandler.fetchHandler); +alertsRouter.post("/notifications/search", setDataToRequestObject("api.alert.notification.list"), checkRBAC.handler(), notificationHandler.listHandler); +alertsRouter.post("/notifications/create", setDataToRequestObject("api.alert.notification.create"), checkRBAC.handler(), notificationHandler.createHandler); +alertsRouter.get("/notifications/publish/:id", setDataToRequestObject("api.alert.notification.publish"), checkRBAC.handler(), notificationHandler.publishHandler); +alertsRouter.post("/notifications/test", setDataToRequestObject("api.alert.notification.test"), checkRBAC.handler(), notificationHandler.testNotifationChannelHandler); +alertsRouter.patch("/notifications/update/:id", setDataToRequestObject("api.alert.notification.update"), checkRBAC.handler(), notificationHandler.updateHandler); +alertsRouter.delete("/notifications/delete/:id", setDataToRequestObject("api.alert.notification.retire"), checkRBAC.handler(), notificationHandler.retireHandler); +alertsRouter.get("/notifications/get/:id", setDataToRequestObject("api.alert.notification.get"), checkRBAC.handler(), notificationHandler.fetchHandler); // alerts -alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), customAlertHandler.createAlertHandler); -alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), customAlertHandler.publishAlertHandler); -alertsRouter.post(`/search`, setDataToRequestObject("api.alert.list"), customAlertHandler.searchAlertHandler); -alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), customAlertHandler.alertDetailsHandler); -alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteAlertHandler); -alertsRouter.delete("/delete", setDataToRequestObject("api.alert.delete"), customAlertHandler.deleteSystemAlertsHandler); -alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), customAlertHandler.updateAlertHandler); +alertsRouter.post("/create", setDataToRequestObject("api.alert.create"), checkRBAC.handler(), customAlertHandler.createAlertHandler); +alertsRouter.get("/publish/:alertId", setDataToRequestObject("api.alert.publish"), checkRBAC.handler(), customAlertHandler.publishAlertHandler); +alertsRouter.post(`/search`, setDataToRequestObject("api.alert.list"), checkRBAC.handler(), customAlertHandler.searchAlertHandler); +alertsRouter.get("/get/:alertId", setDataToRequestObject("api.alert.getAlertDetails"), checkRBAC.handler(), customAlertHandler.alertDetailsHandler); +alertsRouter.delete("/delete/:alertId", setDataToRequestObject("api.alert.delete"), checkRBAC.handler(), customAlertHandler.deleteAlertHandler); +alertsRouter.delete("/delete", setDataToRequestObject("api.alert.delete"), checkRBAC.handler(), customAlertHandler.deleteSystemAlertsHandler); +alertsRouter.patch("/update/:alertId", setDataToRequestObject("api.alert.update"), checkRBAC.handler(), customAlertHandler.updateAlertHandler); // metrics -alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), metricAliasHandler.createMetricHandler); -alertsRouter.post("/metric/alias/search", setDataToRequestObject("api.metric.list"), metricAliasHandler.listMetricsHandler); -alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"),metricAliasHandler.updateMetricHandler); -alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"),metricAliasHandler.deleteMetricHandler); -alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), metricAliasHandler.deleteMultipleMetricHandler); +alertsRouter.post("/metric/alias/create",setDataToRequestObject("api.metric.add"), checkRBAC.handler(), metricAliasHandler.createMetricHandler); +alertsRouter.post("/metric/alias/search", setDataToRequestObject("api.metric.list"), checkRBAC.handler(), metricAliasHandler.listMetricsHandler); +alertsRouter.patch("/metric/alias/update/:id", setDataToRequestObject("api.metric.update"), checkRBAC.handler(), metricAliasHandler.updateMetricHandler); +alertsRouter.delete("/metric/alias/delete/:id", setDataToRequestObject("api.metric.remove"), checkRBAC.handler(), metricAliasHandler.deleteMetricHandler); +alertsRouter.delete("/metric/alias/delete", setDataToRequestObject("api.metric.remove"), checkRBAC.handler(), metricAliasHandler.deleteMultipleMetricHandler); // silence -alertsRouter.post("/silence/create",setDataToRequestObject("api.alert.silence.create"),silenceHandler.createHandler); -alertsRouter.get("/silence/search",setDataToRequestObject("api.alert.silence.list"),silenceHandler.listHandler); -alertsRouter.get("/silence/get/:id",setDataToRequestObject("api.alert.silence.get"),silenceHandler.fetchHandler); -alertsRouter.patch("/silence/update/:id",setDataToRequestObject("api.alert.silence.edit"),silenceHandler.updateHandler); -alertsRouter.delete("/silence/delete/:id",setDataToRequestObject("api.alert.silence.delete"),silenceHandler.deleteHandler); \ No newline at end of file +alertsRouter.post("/silence/create",setDataToRequestObject("api.alert.silence.create"), checkRBAC.handler(), silenceHandler.createHandler); +alertsRouter.get("/silence/search",setDataToRequestObject("api.alert.silence.list"), checkRBAC.handler(), silenceHandler.listHandler); +alertsRouter.get("/silence/get/:id",setDataToRequestObject("api.alert.silence.get"), checkRBAC.handler(), silenceHandler.fetchHandler); +alertsRouter.patch("/silence/update/:id",setDataToRequestObject("api.alert.silence.edit"), checkRBAC.handler(), silenceHandler.updateHandler); +alertsRouter.delete("/silence/delete/:id",setDataToRequestObject("api.alert.silence.delete"), checkRBAC.handler(), silenceHandler.deleteHandler); \ No newline at end of file From 02bd46b7219a73735ea84c0296dc27960721ae2f Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 10:29:57 +0530 Subject: [PATCH 208/235] #OBS-I165: modified the user permissions into a json object --- .../src/middlewares/RBAC_middleware.ts | 346 ++++++++---------- 1 file changed, 147 insertions(+), 199 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index e37bd916..f9f615c6 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -4,205 +4,144 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; import { config } from "../configs/Config"; import _ from "lodash"; -enum roles { - OperationsAdmin = "operations_admin", - Admin = "admin", - DatasetManager = "dataset_manager", - Viewer = "viewer", - DatasetCreator = "dataset_creator", - Ingestor = "ingestor", -} - -enum permissions { - AlertCreate = "api.alert.create", - AlertPublish = "api.alert.publish", - AlertUpdate = "api.alert.update", - AlertList = "api.alert.list", - AlertDelete = "api.alert.delete", - AlertDetails = "api.alert.getAlertDetails", - MetricCreate = "api.metric.add", - MetricList = "api.metric.list", - MetricUpdate = "api.metric.update", - MetricRemove = "api.metric.remove", - SilenceCreate = "api.alert.silence.create", - SilenceList = "api.alert.silence.list", - SilenceRead = "api.alert.silence.get", - SilenceEdit = "api.alert.silence.edit", - SilenceDelete = "api.alert.silence.delete", - NotificationChannelCreate = "api.alert.notification.create", - NotificationChannelList = "api.alert.notification.list", - NotificationChannelPublish = "api.alert.notification.publish", - NotificationChannelTest = "api.alert.notification.test", - NotificationChannelUpdate = "api.alert.notification.update", - NotificationChannelRetire = "api.alert.notification.retire", - NotificationChannelRead = "api.alert.notification.get", - DatasetCreate = "api.datasets.create", - DatasetUpdate = "api.datasets.update", - DatasetRead = "api.datasets.read", - DatasetList = "api.datasets.list", - DataIngest = "api.data.in", - DataOut = "api.data.out", - DataExhaust = "api.data.exhaust", - QueryTemplateCreate = "api.query.template.create", - QueryTemplateRead = "api.query.template.read", - QueryTemplateDelete = "api.query.template.delete", - QueryTemplateList = "api.query.template.list", - QueryTemplateUpdate = "api.query.template.update", - QueryTemplate = "api.query.template.query", - SchemaValidator = "api.schema.validator", - GenerateSignedUrl = "api.files.generate-url", - DatasetStatusTransition = "api.datasets.status-transition", - DatasetHealth = "api.dataset.health", - DatasetReset = "api.dataset.reset", - DatasetSchemaGenerator = "api.datasets.dataschema", - DatasetExport = "api.datasets.export", - DatasetCopy = "api.datasets.copy", - ConnectorList = "api.connectors.list", - ConnectorRead = "api.connectors.read", - DatasetImport = "api.datasets.import", - SQLQuery = "api.obsrv.data.sql-query", -} - interface AccessControl { - [key: string]: string[]; + apiGroups : { + [key: string]: string[]; + }, + roles : { + [key: string]: string[]; + } } const accessControl: AccessControl = { - [roles.Ingestor]: [permissions.DataIngest], - [roles.Viewer]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.AlertList, - permissions.AlertDetails, - permissions.MetricList, - permissions.SilenceList, - permissions.SilenceRead, - permissions.NotificationChannelList, - permissions.NotificationChannelRead, - ], - [roles.DatasetCreator]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.QueryTemplateUpdate, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.AlertList, - permissions.AlertDetails, - permissions.MetricList, - permissions.SilenceList, - permissions.SilenceRead, - permissions.NotificationChannelList, - permissions.NotificationChannelRead, - ], - [roles.DatasetManager]: [ - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.QueryTemplateUpdate, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.DatasetReset, - permissions.DatasetStatusTransition, - permissions.AlertList, - permissions.AlertDetails, - permissions.MetricList, - permissions.SilenceList, - permissions.SilenceRead, - permissions.NotificationChannelList, - permissions.NotificationChannelRead, - ], - [roles.Admin]: [ - permissions.DatasetCreate, - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - permissions.DatasetImport, - permissions.DatasetCreate, - permissions.DatasetUpdate, - permissions.DatasetCopy, - permissions.QueryTemplateCreate, - permissions.QueryTemplateRead, - permissions.QueryTemplateDelete, - permissions.QueryTemplateUpdate, - permissions.GenerateSignedUrl, - permissions.SchemaValidator, - permissions.DatasetSchemaGenerator, - permissions.DatasetReset, - permissions.DatasetStatusTransition, - permissions.AlertList, - permissions.AlertDetails, - permissions.MetricList, - permissions.SilenceList, - permissions.SilenceRead, - permissions.NotificationChannelList, - permissions.NotificationChannelRead, - ], - [roles.OperationsAdmin]: [ - permissions.AlertCreate, - permissions.AlertPublish, - permissions.AlertUpdate, - permissions.AlertList, - permissions.AlertDelete, - permissions.AlertDetails, - permissions.MetricCreate, - permissions.MetricList, - permissions.MetricUpdate, - permissions.MetricRemove, - permissions.SilenceCreate, - permissions.SilenceList, - permissions.SilenceRead, - permissions.SilenceEdit, - permissions.SilenceDelete, - permissions.NotificationChannelCreate, - permissions.NotificationChannelList, - permissions.NotificationChannelPublish, - permissions.NotificationChannelTest, - permissions.NotificationChannelUpdate, - permissions.NotificationChannelRetire, - permissions.NotificationChannelRead, - permissions.DatasetList, - permissions.DatasetRead, - permissions.DatasetExport, - permissions.ConnectorRead, - permissions.SQLQuery, - permissions.DataOut, - permissions.DataExhaust, - ], -}; + "apiGroups": { + "general_access": [ + "api.datasets.list", + "api.datasets.read", + "api.datasets.export", + "api.data.out", + "api.data.exhaust", + "api.alert.list", + "api.alert.getAlertDetails", + "api.metric.list", + "api.alert.silence.list", + "api.alert.silence.get", + "api.alert.notification.list", + "api.alert.notification.get" + ], + "restricted_dataset_api": [ + "api.datasets.reset", + "api.datasets.status-transition" + ], + "alert": [ + "api.alert.create", + "api.alert.publish", + "api.alert.update", + "api.alert.delete" + ], + "metric": [ + "api.metric.add", + "api.metric.update", + "api.metric.remove" + ], + "silence": [ + "api.alert.silence.create", + "api.alert.silence.edit", + "api.alert.silence.delete" + ], + "notificationChannel": [ + "api.alert.notification.create", + "api.alert.notification.publish", + "api.alert.notification.test", + "api.alert.notification.update", + "api.alert.notification.retire" + ], + "dataset": [ + "api.datasets.create", + "api.datasets.update", + "api.datasets.import", + "api.datasets.copy", + "api.dataset.health", + "api.datasets.dataschema" + ], + "data": [ + "api.data.in" + ], + "queryTemplate": [ + "api.query.template.create", + "api.query.template.read", + "api.query.template.delete", + "api.query.template.update", + "api.query.template.query", + "api.query.template.list" + ], + "schema": [ + "api.schema.validator" + ], + "file": [ + "api.files.generate-url" + ], + "connector": [ + "api.connectors.list", + "api.connectors.read" + ], + "sqlQuery": [ + "api.obsrv.data.sql-query" + ] + }, + "roles": { + "ingestor": [ + "data" + ], + "viewer": [ + "general_access", + "connector", + "sqlQuery" + ], + "dataset_creator": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery" + ], + "dataset_manager": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "admin": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "operations_admin": [ + "alert", + "metric", + "silence", + "notificationChannel", + "general_access" + ] + } +} export default { name: "rbac:middleware", @@ -239,10 +178,19 @@ export default { if (decoded && _.isObject(decoded)) { (req as any).userInfo = decoded; const action = (req as any).id; - const hasAccess = decoded?.roles?.some( - (role: string) => - accessControl[role] && accessControl[role].includes(action) - ); + // const hasAccess = decoded?.roles?.some( + // (role: string) => + // accessControl[role] && accessControl[role].includes(action) + // ); + const hasAccess = decoded?.roles?.some((role: string) => { + const apiGroups = accessControl.roles[role]; + + if (!apiGroups) return false; + + return apiGroups.some((apiGroup: string) => + accessControl.apiGroups[apiGroup]?.includes(action) + ); + }); if (!hasAccess) { return ResponseHandler.errorResponse( { From 998a846cbef048ebf4144b84c0f5de6131a21c0b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 11:14:33 +0530 Subject: [PATCH 209/235] #OBS-I165: removed unused code --- api-service/src/middlewares/RBAC_middleware.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index f9f615c6..1fc951ac 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -178,10 +178,6 @@ export default { if (decoded && _.isObject(decoded)) { (req as any).userInfo = decoded; const action = (req as any).id; - // const hasAccess = decoded?.roles?.some( - // (role: string) => - // accessControl[role] && accessControl[role].includes(action) - // ); const hasAccess = decoded?.roles?.some((role: string) => { const apiGroups = accessControl.roles[role]; From 9dc9e492ff42529dd2b9f337c121ea49df775491 Mon Sep 17 00:00:00 2001 From: Santhosh Vasabhaktula Date: Thu, 12 Sep 2024 12:33:18 +0530 Subject: [PATCH 210/235] #OBS-I203: Remove the archived and purged status transition from the API --- .../DatasetStatusTransition.ts | 24 ++++--------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index c2a18021..c97b19c5 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -18,11 +18,9 @@ const allowedTransitions: Record = { Delete: [DatasetStatus.Draft, DatasetStatus.ReadyToPublish], ReadyToPublish: [DatasetStatus.Draft], Live: [DatasetStatus.ReadyToPublish], - Retire: [DatasetStatus.Live], - Archive: [DatasetStatus.Retired], - Purge: [DatasetStatus.Archived] + Retire: [DatasetStatus.Live] } -const liveDatasetActions = ["Retire", "Archive", "Purge"] +const liveDatasetActions = ["Retire"] const validateRequest = (req: Request, datasetId: any) => { const isRequestValid: Record = schemaValidation(req.body, StatusTransitionSchema) @@ -70,12 +68,8 @@ const datasetStatusTransition = async (req: Request, res: Response) => { case "Retire": await retireDataset(dataset); break; - case "Archive": - await archiveDataset(dataset); - break; - case "Purge": - await purgeDataset(dataset); - break; + default: + throw obsrvError(dataset.id, "UNKNOWN_STATUS_TRANSITION", "Unknown status transition requested", "BAD_REQUEST", 400) } ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: `Dataset status transition to ${status} successful`, dataset_id } }); @@ -239,14 +233,4 @@ export const restartPipeline = async (dataset: Record) => { return executeCommand(dataset.id, "RESTART_PIPELINE") } -const archiveDataset = async (dataset: Record) => { - - throw obsrvError(dataset.id, "ARCHIVE_NOT_IMPLEMENTED", "Archive functionality is not implemented", "NOT_IMPLEMENTED", 501) -} - -const purgeDataset = async (dataset: Record) => { - - throw obsrvError(dataset.id, "PURGE_NOT_IMPLEMENTED", "Purge functionality is not implemented", "NOT_IMPLEMENTED", 501) -} - export default datasetStatusTransition; \ No newline at end of file From a2e8a24b562e7eb8c7a74e609a4945f0002a4e0f Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 16:07:04 +0530 Subject: [PATCH 211/235] #OBS-I165: added user permissions json --- .../src/middlewares/userPermissions.json | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 api-service/src/middlewares/userPermissions.json diff --git a/api-service/src/middlewares/userPermissions.json b/api-service/src/middlewares/userPermissions.json new file mode 100644 index 00000000..400e218f --- /dev/null +++ b/api-service/src/middlewares/userPermissions.json @@ -0,0 +1,129 @@ +{ + "apiGroups": { + "general_access": [ + "api.datasets.list", + "api.datasets.read", + "api.datasets.export", + "api.data.out", + "api.data.exhaust", + "api.alert.list", + "api.alert.getAlertDetails", + "api.metric.list", + "api.alert.silence.list", + "api.alert.silence.get", + "api.alert.notification.list", + "api.alert.notification.get" + ], + "restricted_dataset_api": [ + "api.datasets.reset", + "api.datasets.status-transition" + ], + "alert": [ + "api.alert.create", + "api.alert.publish", + "api.alert.update", + "api.alert.delete" + ], + "metric": [ + "api.metric.add", + "api.metric.update", + "api.metric.remove" + ], + "silence": [ + "api.alert.silence.create", + "api.alert.silence.edit", + "api.alert.silence.delete" + ], + "notificationChannel": [ + "api.alert.notification.create", + "api.alert.notification.publish", + "api.alert.notification.test", + "api.alert.notification.update", + "api.alert.notification.retire" + ], + "dataset": [ + "api.datasets.create", + "api.datasets.update", + "api.datasets.import", + "api.datasets.copy", + "api.dataset.health", + "api.datasets.dataschema" + ], + "data": [ + "api.data.in" + ], + "queryTemplate": [ + "api.query.template.create", + "api.query.template.read", + "api.query.template.delete", + "api.query.template.update", + "api.query.template.query", + "api.query.template.list" + ], + "schema": [ + "api.schema.validator" + ], + "file": [ + "api.files.generate-url" + ], + "connector": [ + "api.connectors.list", + "api.connectors.read" + ], + "sqlQuery": [ + "api.obsrv.data.sql-query" + ] + }, + "roles": { + "ingestor": [ + "data" + ], + "viewer": [ + "general_access", + "connector", + "sqlQuery" + ], + "dataset_creator": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery" + ], + "dataset_manager": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "admin": [ + "general_access", + "connector", + "sqlQuery", + "dataset", + "queryTemplate", + "schema", + "file", + "connector", + "sqlQuery", + "restricted_dataset_api" + ], + "operations_admin": [ + "alert", + "metric", + "silence", + "notificationChannel", + "general_access" + ] + } +} \ No newline at end of file From 179e8dcdcc6e31ce6a9213470477ec42d0d49a59 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 16:07:43 +0530 Subject: [PATCH 212/235] #OBS-I165: removed user roles for type --- api-service/src/services/telemetry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index f24995a9..b419db2a 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -18,7 +18,7 @@ const getDefaults = (userInfo:any) => { mid: v4(), actor: { id: userInfo?.id || "SYSTEM", - type: userInfo?.roles[0] || "User", + type: "User", }, context: { env, From 6a0bc426c2cf55ce1e738b8051f46b3556d91f40 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 16:09:13 +0530 Subject: [PATCH 213/235] #OBS-I165: added userID to req object and importing permissions from jsonfile --- .../src/middlewares/RBAC_middleware.ts | 134 +----------------- 1 file changed, 3 insertions(+), 131 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 1fc951ac..60e1ab72 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -3,7 +3,7 @@ import jwt from "jsonwebtoken"; import { ResponseHandler } from "../helpers/ResponseHandler"; import { config } from "../configs/Config"; import _ from "lodash"; - +import userPermissions from "./userPermissions.json"; interface AccessControl { apiGroups : { [key: string]: string[]; @@ -13,135 +13,7 @@ interface AccessControl { } } -const accessControl: AccessControl = { - "apiGroups": { - "general_access": [ - "api.datasets.list", - "api.datasets.read", - "api.datasets.export", - "api.data.out", - "api.data.exhaust", - "api.alert.list", - "api.alert.getAlertDetails", - "api.metric.list", - "api.alert.silence.list", - "api.alert.silence.get", - "api.alert.notification.list", - "api.alert.notification.get" - ], - "restricted_dataset_api": [ - "api.datasets.reset", - "api.datasets.status-transition" - ], - "alert": [ - "api.alert.create", - "api.alert.publish", - "api.alert.update", - "api.alert.delete" - ], - "metric": [ - "api.metric.add", - "api.metric.update", - "api.metric.remove" - ], - "silence": [ - "api.alert.silence.create", - "api.alert.silence.edit", - "api.alert.silence.delete" - ], - "notificationChannel": [ - "api.alert.notification.create", - "api.alert.notification.publish", - "api.alert.notification.test", - "api.alert.notification.update", - "api.alert.notification.retire" - ], - "dataset": [ - "api.datasets.create", - "api.datasets.update", - "api.datasets.import", - "api.datasets.copy", - "api.dataset.health", - "api.datasets.dataschema" - ], - "data": [ - "api.data.in" - ], - "queryTemplate": [ - "api.query.template.create", - "api.query.template.read", - "api.query.template.delete", - "api.query.template.update", - "api.query.template.query", - "api.query.template.list" - ], - "schema": [ - "api.schema.validator" - ], - "file": [ - "api.files.generate-url" - ], - "connector": [ - "api.connectors.list", - "api.connectors.read" - ], - "sqlQuery": [ - "api.obsrv.data.sql-query" - ] - }, - "roles": { - "ingestor": [ - "data" - ], - "viewer": [ - "general_access", - "connector", - "sqlQuery" - ], - "dataset_creator": [ - "general_access", - "connector", - "sqlQuery", - "dataset", - "queryTemplate", - "schema", - "file", - "connector", - "sqlQuery" - ], - "dataset_manager": [ - "general_access", - "connector", - "sqlQuery", - "dataset", - "queryTemplate", - "schema", - "file", - "connector", - "sqlQuery", - "restricted_dataset_api" - ], - "admin": [ - "general_access", - "connector", - "sqlQuery", - "dataset", - "queryTemplate", - "schema", - "file", - "connector", - "sqlQuery", - "restricted_dataset_api" - ], - "operations_admin": [ - "alert", - "metric", - "silence", - "notificationChannel", - "general_access" - ] - } -} +const accessControl: AccessControl = userPermissions; export default { name: "rbac:middleware", @@ -176,7 +48,7 @@ export default { ); } if (decoded && _.isObject(decoded)) { - (req as any).userInfo = decoded; + (req as any).userID = decoded?.id; const action = (req as any).id; const hasAccess = decoded?.roles?.some((role: string) => { const apiGroups = accessControl.roles[role]; From 52273e9f7f5a50a35e33eabeb38ee16cd609144b Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 16:11:12 +0530 Subject: [PATCH 214/235] #OBS-I165: handled rbac disabled scenario and updated userID instead of userRole --- api-service/src/controllers/Alerts/Alerts.ts | 13 ++++++++----- api-service/src/controllers/Alerts/Silence.ts | 4 ++-- .../CreateQueryTemplate/CreateTemplateController.ts | 3 ++- .../src/controllers/DatasetCopy/DatasetCopy.ts | 2 +- .../src/controllers/DatasetCreate/DatasetCreate.ts | 2 +- .../src/controllers/DatasetImport/DatasetImport.ts | 2 +- .../src/controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition.ts | 2 +- .../src/controllers/DatasetUpdate/DatasetUpdate.ts | 3 ++- .../controllers/NotificationChannel/Notification.ts | 8 ++++---- .../UpdateQueryTemplate/UpdateTemplateController.ts | 3 ++- 11 files changed, 25 insertions(+), 19 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 131fc9a2..90142188 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -13,7 +13,8 @@ const telemetryObject = { type: "alert", ver: "1.0.0" }; const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const alertPayload = getAlertPayload(req.body); - _.set(alertPayload, "created_by", (req as any)?.userInfo?.roles[0]); + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(alertPayload, "created_by", userRole); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); @@ -31,7 +32,8 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const { alertId } = req.params; const rulePayload: Record | null = await getAlertRule(alertId); if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); - _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(rulePayload, "updated_by", userRole); if (rulePayload.status == "live") { await deleteAlertRule(rulePayload, false); } @@ -89,7 +91,8 @@ const deleteAlertHandler = async (req: Request, res: Response, next: NextFunctio return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); } const rulePayload = ruleModel.toJSON(); - _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(rulePayload, "updated_by", userRole); await deleteAlertRule(rulePayload, hardDelete === "true"); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); @@ -106,13 +109,13 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio const ruleModel = await getAlertRule(alertId); if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } const rulePayload = ruleModel.toJSON(); + const userRole = (req as any)?.userID || "SYSTEM"; if (rulePayload.status == "live") { - _.set(rulePayload, "updated_by", (req as any)?.userInfo?.roles[0]); + _.set(rulePayload, "updated_by", userRole); await deleteAlertRule(rulePayload, false); await retireAlertSilence(alertId); } const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); - const userRole = (req as any)?.userInfo?.roles[0]; await Alert.update({ ...updatedPayload, status: "draft", updated_by: userRole }, { where: { id: alertId } }); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index fd2bee31..f0ae97d0 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -20,7 +20,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun const start_date = new Date(startDate); const end_date = new Date(endDate); - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, @@ -80,7 +80,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun await updateSilence(silenceObject, payload); const updatedStartTime = new Date(payload.startTime); const updatedEndTime = new Date(payload.endTime); - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID; const updatedSilence = { ...silenceObject, start_time: updatedStartTime, diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index 94b4cffd..2a7d861c 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -42,7 +42,8 @@ export const createQueryTemplate = async (req: Request, res: Response) => { } const data = transformRequest(requestBody, templateName); - _.set(data, "created_by", (req as any)?.userInfo?.roles[0]); + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(data, "created_by", userRole); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) return ResponseHandler.successResponse(req, res, { status: 200, data: { template_id: templateId, template_name: templateName, message: `The query template has been saved successfully` } }); diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 8ddc9fe2..0d803ee0 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -40,7 +40,7 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); - const userRole = (req as any)?.userInfo?.roles[0]; + const userRole = (req as any)?.userID || "SYSTEM"; _.set(dataset, "created_by", userRole); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 2e476b7c..550413b1 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -34,7 +34,7 @@ const datasetCreate = async (req: Request, res: Response) => { await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) - const userRole = (req as any)?.userInfo?.roles[0] || "SYSTEM"; + const userRole = (req as any)?.userID || "SYSTEM"; _.set(draftDataset, "created_by", userRole); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 97314ecc..6ff8f09d 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -15,7 +15,7 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } - const userRole = (req as any)?.userInfo?.roles[0]; + const userRole = (req as any)?.userID || "SYSTEM"; _.set(datasetPayload, "created_by", userRole); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 5e3a869c..49487014 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -30,7 +30,7 @@ const datasetRead = async (req: Request, res: Response) => { validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; - const userRole = (req as any)?.userInfo?.roles[0]; + const userRole = (req as any)?.userID || "SYSTEM"; const attributes = !fields ? defaultFields : _.split(fields, ","); const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userRole) : await readDataset(dataset_id, attributes) if (!dataset) { diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index eb03c7b5..d5ce4ee8 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -55,7 +55,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) - const userRole = (req as any)?.userInfo?.roles[0] || "SYSTEM"; + const userRole = (req as any)?.userID || "SYSTEM"; validateDataset(dataset, dataset_id, status); switch (status) { diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 679b3fcc..d83962fa 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -59,7 +59,8 @@ const datasetUpdate = async (req: Request, res: Response) => { validateDataset(datasetModel, req) const draftDataset = mergeDraftDataset(datasetModel, datasetReq); - _.set(draftDataset, "updated_by", (req as any)?.userInfo?.roles[0] ) + const userRole = (req as any)?.userID || "SYSTEM"; + _.set(draftDataset, "updated_by", userRole ) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index bc470c36..21d4ae74 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,7 +12,7 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; _.set(payload, "created_by", userRole); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); @@ -34,7 +34,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; _.set(updatedPayload, "updated_by", userRole); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); @@ -78,7 +78,7 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; await Notification.update({ status: "retired", updated_by: userRole }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -96,7 +96,7 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - const userRole = (request as any)?.userInfo?.roles[0]; + const userRole = (request as any)?.userID || "SYSTEM"; Notification.update({ status: "live", updated_by: userRole }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { diff --git a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts index 30f3c5c5..741b2114 100644 --- a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts +++ b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts @@ -38,7 +38,8 @@ export const updateQueryTemplate = async (req: Request, res: Response) => { logger.error({ apiId, msgid, resmsgid, templateId, requestBody: req?.body, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, code: "QUERY_TEMPLATE_INVALID_INPUT" }) return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, errCode: "BAD_REQUEST", code: "QUERY_TEMPLATE_INVALID_INPUT" }, req, res) } - requestBody.request.updated_by = (req as any)?.userInfo?.roles[0]; + const userRole = (req as any)?.userID || "SYSTEM"; + requestBody.request.updated_by = userRole; await QueryTemplate.update(requestBody?.request, { where: { template_id: templateId } }) logger.info({ apiId, msgid, resmsgid, templateId, requestBody, message: `Query template updated successfully` }) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Query template updated successfully", templateId } }); From 2eac48fc4308d3437861afa6a287c2d63c06c348 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 18:01:08 +0530 Subject: [PATCH 215/235] #OBS-I165: changed the userRole to userID --- api-service/src/controllers/Alerts/Alerts.ts | 18 +++++++++--------- api-service/src/controllers/Alerts/Silence.ts | 8 ++++---- .../CreateTemplateController.ts | 4 ++-- .../src/controllers/DatasetCopy/DatasetCopy.ts | 4 ++-- .../controllers/DatasetCreate/DatasetCreate.ts | 4 ++-- .../controllers/DatasetImport/DatasetImport.ts | 10 +++++----- .../src/controllers/DatasetRead/DatasetRead.ts | 10 +++++----- .../DatasetStatusTransition.ts | 12 ++++++------ .../controllers/DatasetUpdate/DatasetUpdate.ts | 4 ++-- .../NotificationChannel/Notification.ts | 16 ++++++++-------- .../UpdateTemplateController.ts | 4 ++-- api-service/src/services/DatasetService.ts | 8 ++++---- api-service/src/services/telemetry.ts | 6 +++--- 13 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 90142188..8e749e22 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -13,8 +13,8 @@ const telemetryObject = { type: "alert", ver: "1.0.0" }; const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const alertPayload = getAlertPayload(req.body); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(alertPayload, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(alertPayload, "created_by", userID); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); @@ -32,8 +32,8 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const { alertId } = req.params; const rulePayload: Record | null = await getAlertRule(alertId); if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(rulePayload, "updated_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(rulePayload, "updated_by", userID); if (rulePayload.status == "live") { await deleteAlertRule(rulePayload, false); } @@ -91,8 +91,8 @@ const deleteAlertHandler = async (req: Request, res: Response, next: NextFunctio return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); } const rulePayload = ruleModel.toJSON(); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(rulePayload, "updated_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(rulePayload, "updated_by", userID); await deleteAlertRule(rulePayload, hardDelete === "true"); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); @@ -109,14 +109,14 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio const ruleModel = await getAlertRule(alertId); if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } const rulePayload = ruleModel.toJSON(); - const userRole = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID || "SYSTEM"; if (rulePayload.status == "live") { - _.set(rulePayload, "updated_by", userRole); + _.set(rulePayload, "updated_by", userID); await deleteAlertRule(rulePayload, false); await retireAlertSilence(alertId); } const updatedPayload = getAlertPayload({ ...req.body, manager: rulePayload?.manager }); - await Alert.update({ ...updatedPayload, status: "draft", updated_by: userRole }, { where: { id: alertId } }); + await Alert.update({ ...updatedPayload, status: "draft", updated_by: userID }, { where: { id: alertId } }); updateTelemetryAuditEvent({ request: req, currentRecord: rulePayload, object: { id: alertId, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: alertId } }); } catch (error: any) { diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index f0ae97d0..4a10730a 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -20,14 +20,14 @@ const createHandler = async (request: Request, response: Response, next: NextFun const start_date = new Date(startDate); const end_date = new Date(endDate); - const userRole = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID || "SYSTEM"; const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, alert_id: alertId, start_time: start_date, end_time: end_date, - created_by : userRole, + created_by : userID, } const sileneResponse = await Silence.create(silenceBody); updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); @@ -80,12 +80,12 @@ const updateHandler = async (request: Request, response: Response, next: NextFun await updateSilence(silenceObject, payload); const updatedStartTime = new Date(payload.startTime); const updatedEndTime = new Date(payload.endTime); - const userRole = (request as any)?.userID; + const userID = (request as any)?.userID; const updatedSilence = { ...silenceObject, start_time: updatedStartTime, end_time: updatedEndTime, - updated_by: userRole, + updated_by: userID, } const silenceResponse = await Silence.update(updatedSilence, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { silenceResponse } }) diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index 2a7d861c..d112bad1 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -42,8 +42,8 @@ export const createQueryTemplate = async (req: Request, res: Response) => { } const data = transformRequest(requestBody, templateName); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(data, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(data, "created_by", userID); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) return ResponseHandler.successResponse(req, res, { status: 200, data: { template_id: templateId, template_name: templateName, message: `The query template has been saved successfully` } }); diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 0d803ee0..8bdf3796 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -40,8 +40,8 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(dataset, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(dataset, "created_by", userID); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { if (err?.name === "SequelizeUniqueConstraintError") { diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 550413b1..e91fb59d 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -34,8 +34,8 @@ const datasetCreate = async (req: Request, res: Response) => { await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(draftDataset, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(draftDataset, "created_by", userID); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 6ff8f09d..ad5b5df8 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -15,21 +15,21 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(datasetPayload, "created_by", userRole); + const userID = (req as any)?.userID || "SYSTEM"; + _.set(datasetPayload, "created_by", userID); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) - const dataset = await importDataset(updatedDataset, overwrite, userRole); + const dataset = await importDataset(updatedDataset, overwrite, userID); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { message: successMsg, data: dataset, ...(!_.isEmpty(partialIgnored) && { ignoredFields: partialIgnored }) } }); } -const importDataset = async (dataset: Record, overwrite: string | any, userRole : string) => { +const importDataset = async (dataset: Record, overwrite: string | any, userID : string) => { const dataset_id = _.get(dataset,"dataset_id") const response = await datasetService.createDraftDataset(dataset).catch(err => { return err }) if (response?.name === "SequelizeUniqueConstraintError") { if (overwrite === "true") { - _.set(dataset, "updated_by", userRole); + _.set(dataset, "updated_by", userID); const overwriteRes = await datasetService.updateDraftDataset(dataset).catch(()=>{ throw obsrvError(dataset_id, "DATASET_IMPORT_FAILURE", `Failed to import dataset: ${dataset_id} as overwrite failed`, "INTERNAL_SERVER_ERROR", 500); }) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 49487014..3de9ab6e 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -30,9 +30,9 @@ const datasetRead = async (req: Request, res: Response) => { validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; - const userRole = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID || "SYSTEM"; const attributes = !fields ? defaultFields : _.split(fields, ","); - const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userRole) : await readDataset(dataset_id, attributes) + const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userID) : await readDataset(dataset_id, attributes) if (!dataset) { throw obsrvError(dataset_id, "DATASET_NOT_FOUND", `Dataset with the given dataset_id:${dataset_id} not found`, "NOT_FOUND", 404); } @@ -42,19 +42,19 @@ const datasetRead = async (req: Request, res: Response) => { ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } -const readDraftDataset = async (datasetId: string, attributes: string[], userRole: string): Promise => { +const readDraftDataset = async (datasetId: string, attributes: string[], userID: string): Promise => { const attrs = _.union(attributes, ["dataset_config", "api_version", "type", "id"]) const draftDataset = await datasetService.getDraftDataset(datasetId, attrs); if (draftDataset) { // Contains a draft const apiVersion = _.get(draftDataset, ["api_version"]); - const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset, userRole) + const dataset: any = (apiVersion === "v2") ? draftDataset : await datasetService.migrateDraftDataset(datasetId, draftDataset, userID) return _.pick(dataset, attributes); } const liveDataset = await datasetService.getDataset(datasetId, undefined, true); if (liveDataset) { - const dataset = await datasetService.createDraftDatasetFromLive(liveDataset, userRole) + const dataset = await datasetService.createDraftDatasetFromLive(liveDataset, userID) return _.pick(dataset, attributes); } diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index d5ce4ee8..041bfd60 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -55,7 +55,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) - const userRole = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID || "SYSTEM"; validateDataset(dataset, dataset_id, status); switch (status) { @@ -63,13 +63,13 @@ const datasetStatusTransition = async (req: Request, res: Response) => { await deleteDataset(dataset); break; case "ReadyToPublish": - await readyForPublish(dataset, userRole); + await readyForPublish(dataset, userID); break; case "Live": - await publishDataset(dataset, userRole); + await publishDataset(dataset, userID); break; case "Retire": - await retireDataset(dataset, userRole); + await retireDataset(dataset, userID); break; case "Archive": await archiveDataset(dataset); @@ -134,10 +134,10 @@ const readyForPublish = async (dataset: Record, updated_by: any) => * * @param dataset */ -const publishDataset = async (dataset: Record, userRole: any) => { +const publishDataset = async (dataset: Record, userID: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record - _.set(draftDataset, ["updated_by"], userRole); + _.set(draftDataset, ["updated_by"], userID); console.log(draftDataset); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index d83962fa..16c9d9a3 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -59,8 +59,8 @@ const datasetUpdate = async (req: Request, res: Response) => { validateDataset(datasetModel, req) const draftDataset = mergeDraftDataset(datasetModel, datasetReq); - const userRole = (req as any)?.userID || "SYSTEM"; - _.set(draftDataset, "updated_by", userRole ) + const userID = (req as any)?.userID || "SYSTEM"; + _.set(draftDataset, "updated_by", userID ) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); } diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index 21d4ae74..5f8e66f1 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,8 +12,8 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; - const userRole = (request as any)?.userID || "SYSTEM"; - _.set(payload, "created_by", userRole); + const userID = (request as any)?.userID || "SYSTEM"; + _.set(payload, "created_by", userID); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) @@ -34,8 +34,8 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } - const userRole = (request as any)?.userID || "SYSTEM"; - _.set(updatedPayload, "updated_by", userRole); + const userID = (request as any)?.userID || "SYSTEM"; + _.set(updatedPayload, "updated_by", userID); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -78,8 +78,8 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - const userRole = (request as any)?.userID || "SYSTEM"; - await Notification.update({ status: "retired", updated_by: userRole }, { where: { id } }) + const userID = (request as any)?.userID || "SYSTEM"; + await Notification.update({ status: "retired", updated_by: userID }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) @@ -96,8 +96,8 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - const userRole = (request as any)?.userID || "SYSTEM"; - Notification.update({ status: "live", updated_by: userRole }, { where: { id } }); + const userID = (request as any)?.userID || "SYSTEM"; + Notification.update({ status: "live", updated_by: userID }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { const error = createError(httpStatus.INTERNAL_SERVER_ERROR, _.get(err, "message") || httpStatus[httpStatus.INTERNAL_SERVER_ERROR]) diff --git a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts index 741b2114..7f7775ae 100644 --- a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts +++ b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts @@ -38,8 +38,8 @@ export const updateQueryTemplate = async (req: Request, res: Response) => { logger.error({ apiId, msgid, resmsgid, templateId, requestBody: req?.body, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, code: "QUERY_TEMPLATE_INVALID_INPUT" }) return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, errCode: "BAD_REQUEST", code: "QUERY_TEMPLATE_INVALID_INPUT" }, req, res) } - const userRole = (req as any)?.userID || "SYSTEM"; - requestBody.request.updated_by = userRole; + const userID = (req as any)?.userID || "SYSTEM"; + requestBody.request.updated_by = userID; await QueryTemplate.update(requestBody?.request, { where: { template_id: templateId } }) logger.info({ apiId, msgid, resmsgid, templateId, requestBody, message: `Query template updated successfully` }) ResponseHandler.successResponse(req, res, { status: 200, data: { message: "Query template updated successfully", templateId } }); diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 4bb27ed7..e7bda7d9 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -92,10 +92,10 @@ class DatasetService { return responseData; } - migrateDraftDataset = async (datasetId: string, dataset: Record, userRole: string): Promise => { + migrateDraftDataset = async (datasetId: string, dataset: Record, userID: string): Promise => { const dataset_id = _.get(dataset, "id") const draftDataset = await this.migrateDatasetV1(dataset_id, dataset); - _.set(draftDataset, "updated_by", userRole); + _.set(draftDataset, "updated_by", userID); const transaction = await sequelize.transaction(); try { await DatasetDraft.update(draftDataset, { where: { id: dataset_id }, transaction }); @@ -167,7 +167,7 @@ class DatasetService { } } - createDraftDatasetFromLive = async (dataset: Model, userRole: string) => { + createDraftDatasetFromLive = async (dataset: Model, userID: string) => { const draftDataset: any = _.omit(dataset, ["created_date", "updated_date", "published_date"]); const dataset_config: any = _.get(dataset, "dataset_config"); @@ -233,7 +233,7 @@ class DatasetService { draftDataset["version_key"] = Date.now().toString() draftDataset["version"] = _.add(_.get(dataset, ["version"]), 1); // increment the dataset version draftDataset["status"] = DatasetStatus.Draft - draftDataset["created_by"] = userRole; + draftDataset["created_by"] = userID; const result = await DatasetDraft.create(draftDataset); return _.get(result, "dataValues") } diff --git a/api-service/src/services/telemetry.ts b/api-service/src/services/telemetry.ts index b419db2a..8408dfb3 100644 --- a/api-service/src/services/telemetry.ts +++ b/api-service/src/services/telemetry.ts @@ -10,14 +10,14 @@ const telemetryTopic = _.get(appConfig, "telemetry_dataset"); export enum OperationType { CREATE = 1, UPDATE, PUBLISH, RETIRE, LIST, GET } -const getDefaults = (userInfo:any) => { +const getDefaults = (userID:any) => { return { eid: "AUDIT", ets: Date.now(), ver: "1.0.0", mid: v4(), actor: { - id: userInfo?.id || "SYSTEM", + id: userID || "SYSTEM", type: "User", }, context: { @@ -128,7 +128,7 @@ export const processAuditEvents = (request: Request) => { _.set(auditEvent, "edata.transition.toState", toState); _.set(auditEvent, "edata.transition.fromState", fromState); } - const telemetryEvent = getDefaults((request as any)?.userInfo); + const telemetryEvent = getDefaults((request as any)?.userID); _.set(telemetryEvent, "edata", edata); _.set(telemetryEvent, "object", { ...(object.id && object.type && { ...object, ver: "1.0.0" }) }); sendTelemetryEvents(telemetryEvent); From 0b6c05dbf99e6dfa48d603c317d4d2c5fbd85da4 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Thu, 12 Sep 2024 19:30:33 +0530 Subject: [PATCH 216/235] #OBS-I165: added createdby for dataset publish api --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 041bfd60..debf430f 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -137,6 +137,7 @@ const readyForPublish = async (dataset: Record, updated_by: any) => const publishDataset = async (dataset: Record, userID: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record + _.set(draftDataset, ["created_by"], userID); _.set(draftDataset, ["updated_by"], userID); console.log(draftDataset); await validateAndUpdateDenormConfig(draftDataset); From 6b68ad2aec357f96a5bd2bc3925bd32928cc34a9 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 16 Sep 2024 11:42:02 +0530 Subject: [PATCH 217/235] #OBS-I165: added error condition and modified status code --- .../src/middlewares/RBAC_middleware.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 60e1ab72..8b073fbe 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -27,8 +27,8 @@ export default { if (!token) { return ResponseHandler.errorResponse( { - statusCode: 403, - errCode: "FORBIDDEN", + statusCode: 401, + errCode: "Unauthorized access", message: "No token provided", }, req, @@ -48,6 +48,17 @@ export default { ); } if (decoded && _.isObject(decoded)) { + if (!decoded?.id) { + return ResponseHandler.errorResponse( + { + statusCode: 401, + errCode: "Unauthorized access", + message: "User ID is missing from the decoded token.", + }, + req, + res + ); + } (req as any).userID = decoded?.id; const action = (req as any).id; const hasAccess = decoded?.roles?.some((role: string) => { @@ -62,8 +73,8 @@ export default { if (!hasAccess) { return ResponseHandler.errorResponse( { - statusCode: 401, - errCode: "Unauthorized access", + statusCode: 403, + errCode: "FORBIDDEN", message: "Access denied for the user", }, req, From 904a526d7adb7726fbee219f53f7db87340ebd28 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Mon, 16 Sep 2024 11:42:53 +0530 Subject: [PATCH 218/235] #OBS-I165: removed redundant code --- api-service/src/controllers/Alerts/Alerts.ts | 6 +++--- api-service/src/controllers/Alerts/Silence.ts | 2 +- .../CreateQueryTemplate/CreateTemplateController.ts | 2 +- api-service/src/controllers/DatasetCopy/DatasetCopy.ts | 2 +- .../src/controllers/DatasetCreate/DatasetCreate.ts | 2 +- .../src/controllers/DatasetImport/DatasetImport.ts | 2 +- api-service/src/controllers/DatasetRead/DatasetRead.ts | 2 +- .../DatasetStatusTransition/DatasetStatusTransition.ts | 2 +- .../src/controllers/DatasetUpdate/DatasetUpdate.ts | 2 +- .../src/controllers/NotificationChannel/Notification.ts | 8 ++++---- .../UpdateQueryTemplate/UpdateTemplateController.ts | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index 8e749e22..eaaef01d 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -13,7 +13,7 @@ const telemetryObject = { type: "alert", ver: "1.0.0" }; const createAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const alertPayload = getAlertPayload(req.body); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(alertPayload, "created_by", userID); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); @@ -32,7 +32,7 @@ const publishAlertHandler = async (req: Request, res: Response, next: NextFuncti const { alertId } = req.params; const rulePayload: Record | null = await getAlertRule(alertId); if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(rulePayload, "updated_by", userID); if (rulePayload.status == "live") { await deleteAlertRule(rulePayload, false); @@ -109,7 +109,7 @@ const updateAlertHandler = async (req: Request, res: Response, next: NextFunctio const ruleModel = await getAlertRule(alertId); if (!ruleModel) { return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }) } const rulePayload = ruleModel.toJSON(); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; if (rulePayload.status == "live") { _.set(rulePayload, "updated_by", userID); await deleteAlertRule(rulePayload, false); diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index 4a10730a..15a7dfda 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -20,7 +20,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun const start_date = new Date(startDate); const end_date = new Date(endDate); - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; const silenceBody = { id: grafanaResponse.silenceId, manager: grafanaResponse.manager, diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index d112bad1..68e36a22 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -42,7 +42,7 @@ export const createQueryTemplate = async (req: Request, res: Response) => { } const data = transformRequest(requestBody, templateName); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(data, "created_by", userID); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 8bdf3796..28707bd2 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -40,7 +40,7 @@ const datasetCopy = async (req: Request, res: Response) => { validateRequest(req); const newDatasetId = _.get(req, "body.request.destination.datasetId"); const dataset = await fetchDataset(req); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(dataset, "created_by", userID); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index e91fb59d..62b2e4b7 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -34,7 +34,7 @@ const datasetCreate = async (req: Request, res: Response) => { await validateRequest(req) const draftDataset = getDraftDataset(req.body.request) - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(draftDataset, "created_by", userID); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index ad5b5df8..6ef907c2 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -15,7 +15,7 @@ const datasetImport = async (req: Request, res: Response) => { const migratedConfigs = migrateExportedDatasetV1(requestBody) datasetPayload = migratedConfigs; } - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(datasetPayload, "created_by", userID); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) diff --git a/api-service/src/controllers/DatasetRead/DatasetRead.ts b/api-service/src/controllers/DatasetRead/DatasetRead.ts index 3de9ab6e..d428b02a 100644 --- a/api-service/src/controllers/DatasetRead/DatasetRead.ts +++ b/api-service/src/controllers/DatasetRead/DatasetRead.ts @@ -30,7 +30,7 @@ const datasetRead = async (req: Request, res: Response) => { validateRequest(req); const { dataset_id } = req.params; const { fields, mode } = req.query; - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; const attributes = !fields ? defaultFields : _.split(fields, ","); const dataset = (mode == "edit") ? await readDraftDataset(dataset_id, attributes, userID) : await readDataset(dataset_id, attributes) if (!dataset) { diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index debf430f..eff474fd 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -55,7 +55,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { validateRequest(req, dataset_id); const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; validateDataset(dataset, dataset_id, status); switch (status) { diff --git a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts index 16c9d9a3..7277b1b8 100644 --- a/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts +++ b/api-service/src/controllers/DatasetUpdate/DatasetUpdate.ts @@ -59,7 +59,7 @@ const datasetUpdate = async (req: Request, res: Response) => { validateDataset(datasetModel, req) const draftDataset = mergeDraftDataset(datasetModel, datasetReq); - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; _.set(draftDataset, "updated_by", userID ) const response = await datasetService.updateDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: response }); diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index 5f8e66f1..deeee234 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -12,7 +12,7 @@ const telemetryObject = { type: "notificationChannel", ver: "1.0.0" }; const createHandler = async (request: Request, response: Response, next: NextFunction) => { try { const payload = request.body; - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; _.set(payload, "created_by", userID); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); @@ -34,7 +34,7 @@ const updateHandler = async (request: Request, response: Response, next: NextFun if (_.get(notificationPayload, "status") === "live") { await updateNotificationChannel(notificationPayload); } - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; _.set(updatedPayload, "updated_by", userID); await Notification.update({ ...updatedPayload, status: "draft" }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); @@ -78,7 +78,7 @@ const retireHandler = async (request: Request, response: Response, next: NextFun if (!notificationPayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await updateNotificationChannel(notificationPayload); - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; await Notification.update({ status: "retired", updated_by: userID }, { where: { id } }) ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id } }); } catch (err) { @@ -96,7 +96,7 @@ const publishHandler = async (request: Request, response: Response, next: NextFu if (notificationPayload.status === "live") throw new Error(httpStatus[httpStatus.CONFLICT]); updateTelemetryAuditEvent({ request, object: { id, ...telemetryObject }, currentRecord: notificationPayload }); await publishNotificationChannel(notificationPayload); - const userID = (request as any)?.userID || "SYSTEM"; + const userID = (request as any)?.userID; Notification.update({ status: "live", updated_by: userID }, { where: { id } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id, status: "published" } }); } catch (err) { diff --git a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts index 7f7775ae..350b25f7 100644 --- a/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts +++ b/api-service/src/controllers/UpdateQueryTemplate/UpdateTemplateController.ts @@ -38,7 +38,7 @@ export const updateQueryTemplate = async (req: Request, res: Response) => { logger.error({ apiId, msgid, resmsgid, templateId, requestBody: req?.body, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, code: "QUERY_TEMPLATE_INVALID_INPUT" }) return ResponseHandler.errorResponse({ statusCode: 400, message: `Invalid template provided, A template should consist of variables ${requiredVariables} and type of json,sql`, errCode: "BAD_REQUEST", code: "QUERY_TEMPLATE_INVALID_INPUT" }, req, res) } - const userID = (req as any)?.userID || "SYSTEM"; + const userID = (req as any)?.userID; requestBody.request.updated_by = userID; await QueryTemplate.update(requestBody?.request, { where: { template_id: templateId } }) logger.info({ apiId, msgid, resmsgid, templateId, requestBody, message: `Query template updated successfully` }) From 0ddb49eca8beb979e3af57ba3318c21437162233 Mon Sep 17 00:00:00 2001 From: yashashk Date: Mon, 16 Sep 2024 11:43:50 +0530 Subject: [PATCH 219/235] #OBS-I222 : Resolved regexp route issue in express version 5 --- api-service/src/app.ts | 2 +- api-service/src/routes/DruidProxyRouter.ts | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api-service/src/app.ts b/api-service/src/app.ts index 27b9fce6..6615ad74 100644 --- a/api-service/src/app.ts +++ b/api-service/src/app.ts @@ -22,7 +22,7 @@ app.use("/v2/", v2Router); app.use("/", druidProxyRouter); app.use("/alerts/v1", alertsRouter); app.use("/", metricRouter); -// app.use("*", ResponseHandler.routeNotFound); +app.use(/(.*)/, ResponseHandler.routeNotFound); app.use(obsrvErrorHandler); app.listen(config.api_port, () => { diff --git a/api-service/src/routes/DruidProxyRouter.ts b/api-service/src/routes/DruidProxyRouter.ts index 70fa3ee6..fdf73315 100644 --- a/api-service/src/routes/DruidProxyRouter.ts +++ b/api-service/src/routes/DruidProxyRouter.ts @@ -9,15 +9,15 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; export const druidProxyRouter = express.Router(); // Send a 410 Gone response to all V1 API calls -// druidProxyRouter.all("/datasets/v1/*", ResponseHandler.goneResponse) -// druidProxyRouter.all("/dataset/v1/*", ResponseHandler.goneResponse) -// druidProxyRouter.all("/datasources/v1/*", ResponseHandler.goneResponse) -// druidProxyRouter.all("/data/v1/*", ResponseHandler.goneResponse) -// druidProxyRouter.all("/template/v1/*", ResponseHandler.goneResponse) +druidProxyRouter.all(/\/datasets\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/dataset\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/datasources\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/data\/v1(.*)/, ResponseHandler.goneResponse); +druidProxyRouter.all(/\/template\/v1(.*)/, ResponseHandler.goneResponse); -// // Druid Proxy APIs for Metabase integration -// druidProxyRouter.post(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative) -// druidProxyRouter.get(/\/druid\/v2.*/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet) -// druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) -// druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) -// druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file +// Druid Proxy APIs for Metabase integration +druidProxyRouter.post(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.native.post"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNative); +druidProxyRouter.get(/\/druid\/v2(.*)/, setDataToRequestObject("query.wrapper.native.get"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeGet); +druidProxyRouter.delete("/druid/v2/:queryId", setDataToRequestObject("query.wrapper.native.delete"), onRequest({ entity: Entity.DruidProxy }), wrapperService.forwardNativeDel) +druidProxyRouter.get("/status", setDataToRequestObject("query.wrapper.status"), onRequest({ entity: Entity.DruidProxy }), wrapperService.nativeStatus) +druidProxyRouter.get("/health", setDataToRequestObject("api.health"), onRequest({ entity: Entity.DruidProxy }), healthService.checkDruidHealth) \ No newline at end of file From 9183129b1134eeae423b536dbd22cf75759b3e3a Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 16 Sep 2024 15:45:08 +0530 Subject: [PATCH 220/235] #OBS-I77 : query api changes to check avialability of datasource --- api-service/src/controllers/DataOut/QueryValidator.ts | 11 +++++++---- .../DataOutTest/DataQueryTest.spec.ts | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 9d79069d..1469e746 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -16,7 +16,7 @@ let dataset_id: string; let requestBody: any; let msgid: string; const errCode = { - notFound: "DATA_OUT_SOURCE_NOT_FOUND", + notFound: "DATASOURCE_NOT_FOUND", invalidDateRange: "DATA_OUT_INVALID_DATE_RANGE" } @@ -175,9 +175,12 @@ const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { const checkSupervisorAvailability = async (datasourceRef: string) => { const { data } = await druidHttpService.get("/druid/coordinator/v1/loadstatus"); - const datasourceLoad = _.get(data, datasourceRef) - if (!(datasourceLoad && datasourceLoad === 100)) { - throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not fully available to query", "RANGE_NOT_SATISFIABLE", 416) + const datasourceAvailaibility = _.get(data, datasourceRef) + if (_.isUndefined(datasourceAvailaibility)) { + throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not available for querying", "NOT_FOUND", 404) + } + if (datasourceAvailaibility !== 100) { + throw obsrvError("", "DATASOURCE_NOT_FULLY_AVAILABLE", "Datasource not fully available for querying", "RANGE_NOT_SATISFIABLE", 416) } } diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index b7af9bbb..2827424b 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -52,7 +52,7 @@ describe("QUERY API TESTS", () => { res.body.params.msgid.should.be.eq(msgid); res.body.responseCode.should.be.eq("NOT_FOUND"); res.body.error.message.should.be.eq("Dataset telemetry-events with table week is not available for querying"); - res.body.error.code.should.be.eq("DATA_OUT_SOURCE_NOT_FOUND"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FOUND"); done(); }); }); @@ -71,7 +71,7 @@ describe("QUERY API TESTS", () => { res.body.responseCode.should.be.eq("NOT_FOUND"); res.body.params.msgid.should.be.eq(msgid); res.body.error.message.should.be.eq("Datasource telemetry-events not available for querying"); - res.body.error.code.should.be.eq("DATA_OUT_SOURCE_NOT_FOUND"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FOUND"); done(); }); }); From 31fbbefc535a42fc9de5931ec65e234b6296a683 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 16 Sep 2024 16:25:46 +0530 Subject: [PATCH 221/235] #OBS-I77 : fix: spell fix --- api-service/src/controllers/DataOut/QueryValidator.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 1469e746..af892572 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -175,11 +175,11 @@ const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { const checkSupervisorAvailability = async (datasourceRef: string) => { const { data } = await druidHttpService.get("/druid/coordinator/v1/loadstatus"); - const datasourceAvailaibility = _.get(data, datasourceRef) - if (_.isUndefined(datasourceAvailaibility)) { + const datasourceAvailability = _.get(data, datasourceRef) + if (_.isUndefined(datasourceAvailability)) { throw obsrvError("", "DATASOURCE_NOT_AVAILABLE", "Datasource not available for querying", "NOT_FOUND", 404) } - if (datasourceAvailaibility !== 100) { + if (datasourceAvailability !== 100) { throw obsrvError("", "DATASOURCE_NOT_FULLY_AVAILABLE", "Datasource not fully available for querying", "RANGE_NOT_SATISFIABLE", 416) } } From 63625bfa2a50548fa3ec0967cf8ccc1adcdef3f0 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Mon, 16 Sep 2024 17:09:41 +0530 Subject: [PATCH 222/235] #OBS-I77 : granularity spell changes --- api-service/src/controllers/DataOut/QueryValidator.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index af892572..45d99049 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -156,7 +156,7 @@ const validateQueryRules = (queryPayload: any, limits: any) => { : { message: "Invalid date range! the date range cannot be a null value", statusCode: 400, errCode: "BAD_REQUEST", code: errCode.invalidDateRange }; }; -const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { +const getDataSourceRef = async (datasetId: string, requestGranularity?: string) => { const dataSources = await getDatasourceList(datasetId) if (_.isEmpty(dataSources)) { logger.error({ apiId, requestBody, msgid, dataset_id, message: `Datasource ${datasetId} not available in datasource live table`, code: errCode.notFound }) @@ -168,7 +168,7 @@ const getDataSourceRef = async (datasetId: string, srcGranularity?: string) => { if (!aggregated) { return true; } - return aggregated && srcGranularity ? granularity === srcGranularity : false; + return aggregated && requestGranularity ? granularity === requestGranularity : false; }); return _.get(record, ["dataValues", "datasource_ref"]) } From fb3fb407d1f7c850576629c980e6cc1c9f99d7aa Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 17 Sep 2024 11:04:17 +0530 Subject: [PATCH 223/235] #OBS-I77 : fix: test scenarios added --- .../src/controllers/DataOut/QueryValidator.ts | 3 + .../DataOutTest/DataQueryTest.spec.ts | 74 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/api-service/src/controllers/DataOut/QueryValidator.ts b/api-service/src/controllers/DataOut/QueryValidator.ts index 45d99049..4c47a90e 100644 --- a/api-service/src/controllers/DataOut/QueryValidator.ts +++ b/api-service/src/controllers/DataOut/QueryValidator.ts @@ -187,6 +187,9 @@ const checkSupervisorAvailability = async (datasourceRef: string) => { const setDatasourceRef = async (datasetId: string, payload: any): Promise => { const granularity = _.get(payload, "context.aggregationLevel") const datasourceRef = await getDataSourceRef(datasetId, granularity); + if (!datasourceRef) { + throw obsrvError("", "DATASOURCE_NOT_FOUND", "Datasource not found to query", "NOT_FOUND", 404) + } await checkSupervisorAvailability(datasourceRef) const existingDatasources = await getDatasourceListFromDruid(); diff --git a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts index 2827424b..ea905305 100644 --- a/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts +++ b/api-service/src/tests/DatasetManagement/DataOutTest/DataQueryTest.spec.ts @@ -19,6 +19,7 @@ const nativeQueryEndpointDruid = config?.query_api?.druid?.native_query_path; const sqlQueryEndpoint = config?.query_api?.druid?.sql_query_path; const response = [{ dataValues: { datasource_ref: "test.1_rollup_week", metadata: { aggregated: true, granularity: "week" } } }] +const invalidResponse = [{ dataValues: { datasource_ref: "test.1_rollup_week", metadata: { aggregated: true, granularity: "n/a" } } }] const msgid = "e180ecac-8f41-4f21-9a21-0b3a1a368917"; describe("QUERY API TESTS", () => { @@ -76,6 +77,79 @@ describe("QUERY API TESTS", () => { }); }); + it("Query api failure: Datasource not available in druid", (done) => { + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve( + response + ) + }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "telemetry_events": 100 } + }) + }) + chai + .request(app) + .post("/v2/data/query/telemetry-events") + .send(JSON.parse(TestQueries.VALID_QUERY)) + .end((err, res) => { + res.should.have.status(404); + res.body.params.status.should.be.eq("FAILED"); + res.body.responseCode.should.be.eq("NOT_FOUND"); + res.body.params.msgid.should.be.eq(msgid); + res.body.error.message.should.be.eq("Datasource not available for querying"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_AVAILABLE"); + done(); + }); + }); + + it("Query api failure: Datasource not fully loaded in druid", (done) => { + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve( + response + ) + }) + chai.spy.on(druidHttpService, "get", () => { + return Promise.resolve({ + data: { "test.1_rollup_week": 20 } + }) + }) + chai + .request(app) + .post("/v2/data/query/telemetry-events") + .send(JSON.parse(TestQueries.VALID_QUERY)) + .end((err, res) => { + res.should.have.status(416); + res.body.params.status.should.be.eq("FAILED"); + res.body.responseCode.should.be.eq("RANGE_NOT_SATISFIABLE"); + res.body.params.msgid.should.be.eq(msgid); + res.body.error.message.should.be.eq("Datasource not fully available for querying"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FULLY_AVAILABLE"); + done(); + }); + }); + + it("Query api failure: Datasource not found", (done) => { + chai.spy.on(Datasource, "findAll", () => { + return Promise.resolve( + invalidResponse + ) + }) + chai + .request(app) + .post("/v2/data/query/telemetry-events") + .send(JSON.parse(TestQueries.VALID_QUERY)) + .end((err, res) => { + res.should.have.status(404); + res.body.params.status.should.be.eq("FAILED"); + res.body.responseCode.should.be.eq("NOT_FOUND"); + res.body.params.msgid.should.be.eq(msgid); + res.body.error.message.should.be.eq("Datasource not found to query"); + res.body.error.code.should.be.eq("DATASOURCE_NOT_FOUND"); + done(); + }); + }); + it("Query api failure : when druid is down, it should raise error when native query endpoint is called", (done) => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve(response) From bb60d4a9f3da48e56d661885a5cf92f612656c3c Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 17 Sep 2024 12:05:20 +0530 Subject: [PATCH 224/235] #OBS-I131 : fix: stop deleting draft records if submission tasks fail --- command-service/src/command/db_command.py | 17 +------------ command-service/src/command/druid_command.py | 24 ++++++++++++++++++- command-service/src/config/service_config.yml | 2 +- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/command-service/src/command/db_command.py b/command-service/src/command/db_command.py index 89db3392..20e9c527 100644 --- a/command-service/src/command/db_command.py +++ b/command-service/src/command/db_command.py @@ -48,7 +48,6 @@ def _change_dataset_to_active(self, command_payload: CommandPayload): self._insert_datasource_record(dataset_id, draft_dataset_id) self._insert_connector_instances(dataset_id, draft_dataset_record) self._insert_dataset_transformations(dataset_id, draft_dataset_record) - self._delete_draft_dataset(dataset_id, draft_dataset_id) return ActionResponse(status="OK", status_code=200) else: return ActionResponse( @@ -412,18 +411,4 @@ def _insert_dataset_transformations(self, dataset_id, draft_dataset_record): """ result = self.db_service.execute_upsert(sql=insert_query, params=params) print(f"Dataset Transformation {dataset_id + '_' + transformation.field_key} record inserted successfully...") - return result - - def _delete_draft_dataset(self, dataset_id, draft_dataset_id): - - self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) - print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") - - self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations_draft where dataset_id = %s""", params=(draft_dataset_id,)) - print(f"Draft transformations/tables for {dataset_id} are deleted successfully...") - - self.db_service.execute_delete(sql=f"""DELETE from dataset_source_config_draft where dataset_id = %s""", params=(draft_dataset_id,)) - print(f"Draft source config/tables for {dataset_id} are deleted successfully...") - - self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) - print(f"Draft Dataset for {dataset_id} is deleted successfully...") \ No newline at end of file + return result \ No newline at end of file diff --git a/command-service/src/command/druid_command.py b/command-service/src/command/druid_command.py index f43222b4..73d5bf86 100644 --- a/command-service/src/command/druid_command.py +++ b/command-service/src/command/druid_command.py @@ -34,6 +34,7 @@ def _submit_ingestion_task(self, dataset_id): print( f"Invoking SUBMIT_INGESTION_TASKS command for dataset_id {dataset_id}..." ) + task_submitted = 1 for record in datasources_records: if record["dataset_type"] == "event" and record["type"] == "druid": print(f"Submitting ingestion task for datasource ...") @@ -41,8 +42,15 @@ def _submit_ingestion_task(self, dataset_id): response = self.http_service.post( url=f"{self.router_url}/{self.supervisor_endpoint}", body=ingestion_spec, - headers={"Content-Type": "application/json"}, + headers={"Content-Type": "application/json"} ) + if response.status != 200: + task_submitted = 0 + break + if task_submitted: + query=f"SELECT id FROM datasets_draft WHERE dataset_id= %s" + response = self.db_service.execute_select_one(sql=query, params=(dataset_id,)) + self._delete_draft_dataset(dataset_id, response[0]) return ActionResponse(status="OK", status_code=200) else: print( @@ -51,3 +59,17 @@ def _submit_ingestion_task(self, dataset_id): return ActionResponse( status="ERROR", status_code=404, error_message="DATASET_ID_NOT_FOUND" ) + + def _delete_draft_dataset(self, dataset_id, draft_dataset_id): + + self.db_service.execute_delete(sql=f"""DELETE from datasources_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft datasources/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_transformations_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft transformations/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from dataset_source_config_draft where dataset_id = %s""", params=(draft_dataset_id,)) + print(f"Draft source config/tables for {dataset_id} are deleted successfully...") + + self.db_service.execute_delete(sql=f"""DELETE from datasets_draft where id = %s""", params=(draft_dataset_id,)) + print(f"Draft Dataset for {dataset_id} is deleted successfully...") diff --git a/command-service/src/config/service_config.yml b/command-service/src/config/service_config.yml index 2c432ba0..8b9f2bdc 100644 --- a/command-service/src/config/service_config.yml +++ b/command-service/src/config/service_config.yml @@ -117,7 +117,7 @@ config_service: port: 4000 druid: - router_host: localhost + router_host: http://localhost router_port: 8888 supervisor_endpoint: indexer/v1/supervisor From 586090e54606cb9d9834f2e978736283cf372235 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 17 Sep 2024 12:42:25 +0530 Subject: [PATCH 225/235] #OBS-I131 : fix: upsert if republished --- api-service/src/services/DatasetService.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index 2e800212..d7679cbe 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -325,7 +325,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "druid"); const ingestionSpec = tableGenerator.getDruidIngestionSpec(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, { transaction }) + await DatasourceDraft.upsert(draftDatasource, { transaction }) } private createHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -334,7 +334,7 @@ class DatasetService { const draftDatasource = this.createDraftDatasource(draftDataset, "hudi"); const ingestionSpec = tableGenerator.getHudiIngestionSpecForCreate(draftDataset, allFields, draftDatasource.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, { transaction }) + await DatasourceDraft.upsert(draftDatasource, { transaction }) } private updateHudiDataSource = async (draftDataset: Record, transaction: Transaction) => { @@ -345,7 +345,7 @@ class DatasetService { const liveDatasource = await Datasource.findOne({ where: { id: dsId }, attributes: ["ingestion_spec"], raw: true }) as unknown as Record const ingestionSpec = tableGenerator.getHudiIngestionSpecForUpdate(draftDataset, liveDatasource?.ingestion_spec, allFields, draftDatasource?.datasource_ref); _.set(draftDatasource, "ingestion_spec", ingestionSpec) - await DatasourceDraft.create(draftDatasource, { transaction }) + await DatasourceDraft.upsert(draftDatasource, { transaction }) } private createDraftDatasource = (draftDataset: Record, type: string): Record => { From 8b560493cfe5e4c8f2c19fdb3fa35794cb87b035 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Tue, 17 Sep 2024 12:43:55 +0530 Subject: [PATCH 226/235] #OBS-I131 : fix: test case fixes --- .../DatasetStatusTransition/DatasetLive.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts index e26ddafe..38dced09 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetLive.spec.ts @@ -38,7 +38,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { @@ -80,7 +80,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { @@ -122,7 +122,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "api_version":"v2", "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) chai.spy.on(Datasource, "findOne", () => { @@ -289,7 +289,7 @@ describe("DATASET STATUS TRANSITION LIVE", () => { chai.spy.on(Dataset, "findOne", () => { return Promise.resolve({ "data_schema": { "email": { "data_type": "string", "arrival_format": "string" } } }) }) - chai.spy.on(DatasourceDraft, "create", () => { + chai.spy.on(DatasourceDraft, "upsert", () => { return Promise.resolve({}) }) const t = chai.spy.on(sequelize, "transaction", () => { From aacd420ad602b22b6368eba98c2fc12f588db8bd Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 12:47:21 +0530 Subject: [PATCH 227/235] #OBS-I165: added default userID, modified error codes, added errorhandler function --- .../src/middlewares/RBAC_middleware.ts | 82 +++++++++++-------- 1 file changed, 46 insertions(+), 36 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 8b073fbe..77f9581f 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -15,49 +15,56 @@ interface AccessControl { const accessControl: AccessControl = userPermissions; +const errorHandler = (statusCode: number, message: string, req: Request, res: Response) => { + let errCode: string; + let code: string; + + switch (statusCode) { + case 401: + errCode = "Unauthorized access"; + code = "UNAUTHORIZED ACCESS"; + break; + case 403: + errCode = "Forbidden access"; + code = "FORBIDDEN ACCESS"; + break; + default: + errCode = "Unknown error"; + code = "UNKNOWN ERROR"; + } + + return ResponseHandler.errorResponse( + { + statusCode, + errCode, + message, + code, + }, + req, + res + ); +}; + export default { name: "rbac:middleware", handler: () => (req: Request, res: Response, next: NextFunction) => { try { if (_.lowerCase(config.is_RBAC_enabled) === "false") { + (req as any).userID = "SYSTEM"; next(); } else { const public_key = config.user_token_public_key; const token = req.get("x-user-token"); if (!token) { - return ResponseHandler.errorResponse( - { - statusCode: 401, - errCode: "Unauthorized access", - message: "No token provided", - }, - req, - res - ); + errorHandler(401, "No token provided", req, res); } jwt.verify(token as string, public_key, (err, decoded) => { if (err) { - return ResponseHandler.errorResponse( - { - statusCode: 403, - errCode: "FORBIDDEN", - message: "Token verification failed", - }, - req, - res - ); + errorHandler(401, "Token verification failed", req, res); } if (decoded && _.isObject(decoded)) { if (!decoded?.id) { - return ResponseHandler.errorResponse( - { - statusCode: 401, - errCode: "Unauthorized access", - message: "User ID is missing from the decoded token.", - }, - req, - res - ); + errorHandler(401, "User ID is missing from the decoded token.", req, res); } (req as any).userID = decoded?.id; const action = (req as any).id; @@ -71,15 +78,18 @@ export default { ); }); if (!hasAccess) { - return ResponseHandler.errorResponse( - { - statusCode: 403, - errCode: "FORBIDDEN", - message: "Access denied for the user", - }, - req, - res - ); + const rolesWithAccess = Object.keys(accessControl.roles).filter(role => { + const apiGroups = accessControl.roles[role]; + if (!apiGroups) return false; + return apiGroups.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); + }); + const rolesMessage = rolesWithAccess.length > 0 + ? `Roles with access: ${rolesWithAccess.join(", ")}` + : "No roles have this action"; + + const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; + + errorHandler(403, errorMessage, req, res); } next(); } From 6e5045291d17309b98f81b0717a2a4143878d0e0 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 12:58:28 +0530 Subject: [PATCH 228/235] #OBS-I165: added updatedby by default when an entry created --- api-service/src/controllers/Alerts/Alerts.ts | 1 + api-service/src/controllers/Alerts/Silence.ts | 1 + .../controllers/CreateQueryTemplate/CreateTemplateController.ts | 1 + api-service/src/controllers/DatasetCopy/DatasetCopy.ts | 1 + api-service/src/controllers/DatasetCreate/DatasetCreate.ts | 1 + api-service/src/controllers/DatasetImport/DatasetImport.ts | 1 + api-service/src/controllers/NotificationChannel/Notification.ts | 1 + 7 files changed, 7 insertions(+) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index eaaef01d..b3bda19e 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -15,6 +15,7 @@ const createAlertHandler = async (req: Request, res: Response, next: NextFunctio const alertPayload = getAlertPayload(req.body); const userID = (req as any)?.userID; _.set(alertPayload, "created_by", userID); + _.set(alertPayload, "updated_by", userID); const response = await Alert.create(alertPayload); updateTelemetryAuditEvent({ request: req, object: { id: response?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: { id: response.dataValues.id } }); diff --git a/api-service/src/controllers/Alerts/Silence.ts b/api-service/src/controllers/Alerts/Silence.ts index 15a7dfda..f424465a 100644 --- a/api-service/src/controllers/Alerts/Silence.ts +++ b/api-service/src/controllers/Alerts/Silence.ts @@ -28,6 +28,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun start_time: start_date, end_time: end_date, created_by : userID, + updated_by : userID, } const sileneResponse = await Silence.create(silenceBody); updateTelemetryAuditEvent({ request, object: { id: sileneResponse?.dataValues?.id, ...telemetryObject } }); diff --git a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts index 68e36a22..c7fffa29 100644 --- a/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts +++ b/api-service/src/controllers/CreateQueryTemplate/CreateTemplateController.ts @@ -44,6 +44,7 @@ export const createQueryTemplate = async (req: Request, res: Response) => { const data = transformRequest(requestBody, templateName); const userID = (req as any)?.userID; _.set(data, "created_by", userID); + _.set(data, "updated_by", userID); await QueryTemplate.create(data) logger.info({ apiId, msgid, resmsgid, requestBody: req?.body, message: `Query template created successfully` }) return ResponseHandler.successResponse(req, res, { status: 200, data: { template_id: templateId, template_name: templateName, message: `The query template has been saved successfully` } }); diff --git a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts index 28707bd2..9015e9c7 100644 --- a/api-service/src/controllers/DatasetCopy/DatasetCopy.ts +++ b/api-service/src/controllers/DatasetCopy/DatasetCopy.ts @@ -42,6 +42,7 @@ const datasetCopy = async (req: Request, res: Response) => { const dataset = await fetchDataset(req); const userID = (req as any)?.userID; _.set(dataset, "created_by", userID); + _.set(dataset, "updated_by", userID); updateRecords(dataset, newDatasetId) const response = await datasetService.createDraftDataset(dataset).catch(err => { if (err?.name === "SequelizeUniqueConstraintError") { diff --git a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts index 62b2e4b7..bb96dac2 100644 --- a/api-service/src/controllers/DatasetCreate/DatasetCreate.ts +++ b/api-service/src/controllers/DatasetCreate/DatasetCreate.ts @@ -36,6 +36,7 @@ const datasetCreate = async (req: Request, res: Response) => { const draftDataset = getDraftDataset(req.body.request) const userID = (req as any)?.userID; _.set(draftDataset, "created_by", userID); + _.set(draftDataset, "updated_by", userID); const dataset = await datasetService.createDraftDataset(draftDataset); ResponseHandler.successResponse(req, res, { status: httpStatus.OK, data: dataset }); } diff --git a/api-service/src/controllers/DatasetImport/DatasetImport.ts b/api-service/src/controllers/DatasetImport/DatasetImport.ts index 6ef907c2..2d0312b5 100644 --- a/api-service/src/controllers/DatasetImport/DatasetImport.ts +++ b/api-service/src/controllers/DatasetImport/DatasetImport.ts @@ -17,6 +17,7 @@ const datasetImport = async (req: Request, res: Response) => { } const userID = (req as any)?.userID; _.set(datasetPayload, "created_by", userID); + _.set(datasetPayload, "updated_by", userID); const { updatedDataset, ignoredFields } = await datasetImportValidation({ ...requestBody, "request": datasetPayload }) const { successMsg, partialIgnored } = getResponseData(ignoredFields) diff --git a/api-service/src/controllers/NotificationChannel/Notification.ts b/api-service/src/controllers/NotificationChannel/Notification.ts index deeee234..d8850436 100644 --- a/api-service/src/controllers/NotificationChannel/Notification.ts +++ b/api-service/src/controllers/NotificationChannel/Notification.ts @@ -14,6 +14,7 @@ const createHandler = async (request: Request, response: Response, next: NextFun const payload = request.body; const userID = (request as any)?.userID; _.set(payload, "created_by", userID); + _.set(payload, "updated_by", userID); const notificationBody = await Notification.create(payload); updateTelemetryAuditEvent({ request, object: { id: notificationBody?.dataValues?.id, ...telemetryObject } }); ResponseHandler.successResponse(request, response, { status: httpStatus.OK, data: { id: notificationBody.dataValues.id } }) From cdb55a7fdcf10b43228ee0b7385645d508c5e230 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 12:59:28 +0530 Subject: [PATCH 229/235] #OBS-I165: removed redundant code --- .../DatasetStatusTransition/DatasetStatusTransition.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index eff474fd..dfc13211 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -139,7 +139,6 @@ const publishDataset = async (dataset: Record, userID: any) => { const draftDataset: Record = await datasetService.getDraftDataset(dataset.dataset_id) as unknown as Record _.set(draftDataset, ["created_by"], userID); _.set(draftDataset, ["updated_by"], userID); - console.log(draftDataset); await validateAndUpdateDenormConfig(draftDataset); await updateMasterDataConfig(draftDataset) await datasetService.publishDataset(draftDataset) From 45f9c617e36fcdd48776fd4d3bf85d150fd23ed1 Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 15:17:36 +0530 Subject: [PATCH 230/235] #OBS-I165: modified errorhandler --- .../src/middlewares/RBAC_middleware.ts | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index 77f9581f..c00199a5 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -4,6 +4,7 @@ import { ResponseHandler } from "../helpers/ResponseHandler"; import { config } from "../configs/Config"; import _ from "lodash"; import userPermissions from "./userPermissions.json"; +import httpStatus from "http-status"; interface AccessControl { apiGroups : { [key: string]: string[]; @@ -16,22 +17,18 @@ interface AccessControl { const accessControl: AccessControl = userPermissions; const errorHandler = (statusCode: number, message: string, req: Request, res: Response) => { - let errCode: string; - let code: string; + const errorMapping: Record = { + 401: { + errCode: httpStatus["401_NAME"], + code: "UNAUTHORIZED ACCESS", + }, + 403: { + errCode: httpStatus["403_NAME"], + code: "FORBIDDEN ACCESS", + }, + }; - switch (statusCode) { - case 401: - errCode = "Unauthorized access"; - code = "UNAUTHORIZED ACCESS"; - break; - case 403: - errCode = "Forbidden access"; - code = "FORBIDDEN ACCESS"; - break; - default: - errCode = "Unknown error"; - code = "UNKNOWN ERROR"; - } + const { errCode, code } = errorMapping[statusCode]; return ResponseHandler.errorResponse( { @@ -56,15 +53,15 @@ export default { const public_key = config.user_token_public_key; const token = req.get("x-user-token"); if (!token) { - errorHandler(401, "No token provided", req, res); + return errorHandler(401, "No token provided", req, res); } jwt.verify(token as string, public_key, (err, decoded) => { if (err) { - errorHandler(401, "Token verification failed", req, res); + return errorHandler(401, "Token verification failed", req, res); } if (decoded && _.isObject(decoded)) { if (!decoded?.id) { - errorHandler(401, "User ID is missing from the decoded token.", req, res); + return errorHandler(401, "User ID is missing from the decoded token.", req, res); } (req as any).userID = decoded?.id; const action = (req as any).id; @@ -89,7 +86,7 @@ export default { const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; - errorHandler(403, errorMessage, req, res); + return errorHandler(403, errorMessage, req, res); } next(); } From 0f148a0607d32580eb39e6d35ee380a04ca4760e Mon Sep 17 00:00:00 2001 From: Rakshitha-D Date: Tue, 17 Sep 2024 17:00:59 +0530 Subject: [PATCH 231/235] #OBS-I165: modified error message --- api-service/src/middlewares/RBAC_middleware.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-service/src/middlewares/RBAC_middleware.ts b/api-service/src/middlewares/RBAC_middleware.ts index c00199a5..b8d4032d 100644 --- a/api-service/src/middlewares/RBAC_middleware.ts +++ b/api-service/src/middlewares/RBAC_middleware.ts @@ -81,7 +81,7 @@ export default { return apiGroups.some(apiGroup => accessControl.apiGroups[apiGroup]?.includes(action)); }); const rolesMessage = rolesWithAccess.length > 0 - ? `Roles with access: ${rolesWithAccess.join(", ")}` + ? `The following roles have access to this action: ${rolesWithAccess.join(", ")}` : "No roles have this action"; const errorMessage = `Access denied. User does not have permission to perform this action. ${rolesMessage}.`; From cd11e2c64cd35ffc55b28ceff709bba1074f99f1 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 18 Sep 2024 15:09:10 +0530 Subject: [PATCH 232/235] #OBS-I227 : fix: threshold of dataset level alerts inside an array. --- command-service/src/command/alert_manager_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/command-service/src/command/alert_manager_command.py b/command-service/src/command/alert_manager_command.py index 1c989a5c..6d478c3e 100644 --- a/command-service/src/command/alert_manager_command.py +++ b/command-service/src/command/alert_manager_command.py @@ -176,7 +176,7 @@ def create_alert_rule(self, payload: dict) -> ActionResponse: "subComponent": dataset_name, "metric": prom_metric, "operator": operator, - "threshold": threshold, + "threshold": [+threshold], "metricAlias": metric_alias, } }, From 4f8dee1f2cc32b058ee135087873d9f675f5c0cd Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 18 Sep 2024 15:11:51 +0530 Subject: [PATCH 233/235] #OBS-I179 : feat: retire dataset related alerts and metrics on dataset retire --- .../DatasetStatusTransition.ts | 2 +- api-service/src/services/DatasetService.ts | 12 ++++++++++-- api-service/src/services/managers/index.ts | 8 ++------ 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts index 43d42ba5..4de7a474 100644 --- a/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts +++ b/api-service/src/controllers/DatasetStatusTransition/DatasetStatusTransition.ts @@ -52,7 +52,7 @@ const datasetStatusTransition = async (req: Request, res: Response) => { const { dataset_id, status } = _.get(req.body, "request"); validateRequest(req, dataset_id); - const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) + const dataset: Record = (_.includes(liveDatasetActions, status)) ? await datasetService.getDataset(dataset_id, ["id", "status", "type", "api_version", "name"], true) : await datasetService.getDraftDataset(dataset_id, ["id", "dataset_id", "status", "type", "api_version"]) const userID = (req as any)?.userID; validateDataset(dataset, dataset_id, status); diff --git a/api-service/src/services/DatasetService.ts b/api-service/src/services/DatasetService.ts index c3284b70..1c864404 100644 --- a/api-service/src/services/DatasetService.ts +++ b/api-service/src/services/DatasetService.ts @@ -17,6 +17,7 @@ import { Datasource } from "../models/Datasource"; import { obsrvError } from "../types/ObsrvError"; import { druidHttpService } from "../connections/druidConnection"; import { tableGenerator } from "./TableGenerator"; +import { deleteAlertByDataset, deleteMetricAliasByDataset } from "./managers"; class DatasetService { @@ -258,6 +259,11 @@ class DatasetService { } } + private deleteAlerts = async (dataset: any) => { + await deleteAlertByDataset(dataset); + await deleteMetricAliasByDataset(dataset); + } + retireDataset = async (dataset: Record, updatedBy: any) => { const transaction = await sequelize.transaction(); @@ -267,11 +273,13 @@ class DatasetService { await Datasource.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); await DatasetTransformations.update({ status: DatasetStatus.Retired, updated_by: updatedBy }, { where: { dataset_id: dataset.id }, transaction }); await transaction.commit(); - await this.deleteDruidSupervisors(dataset); } catch (err: any) { await transaction.rollback(); throw obsrvError(dataset.id, "FAILED_TO_RETIRE_DATASET", err.message, "SERVER_ERROR", 500, err); } + // Deleting dataset alerts and druid supervisors + await this.deleteDruidSupervisors(dataset); + await this.deleteAlerts(dataset); } findDatasources = async (where?: Record, attributes?: string[], order?: any): Promise => { @@ -379,7 +387,7 @@ export const getLiveDatasetConfigs = async (dataset_id: string) => { const transformations = await datasetService.getTransformations(dataset_id, ["field_key", "transformation_function", "mode"]) const connectorsV2 = await datasetService.getConnectors(dataset_id, ["id", "connector_id", "connector_config", "operations_config"]) const connectorsV1 = await getV1Connectors(dataset_id) - const connectors = _.concat(connectorsV1,connectorsV2) + const connectors = _.concat(connectorsV1, connectorsV2) if (!_.isEmpty(transformations)) { datasetRecord["transformations_config"] = transformations diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index e6cf822f..43e796dc 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -7,7 +7,7 @@ import { Metrics } from "../../models/Metric"; import constants from "./constants"; export const getAlertRule = (id: string) => { - return Alert.findOne({ where: { id } }); + return Alert.findOne({ where: { id }, raw: true }); } const getService = (manager: string) => { @@ -138,11 +138,7 @@ export const deleteAlertByDataset = async (payload: Record) => { export const deleteMetricAliasByDataset = async (payload: Record) => { try { const { name } = payload; - const metricAliasPayload = await Metrics.findAll({ where: { component: "datasets", subComponent: name } }) - if (!metricAliasPayload) throw new Error(constants.METRIC_ALIAS_NOT_FOUND) - for (const payload of metricAliasPayload) { - await payload.destroy() - } + await Metrics.destroy({ where: { component: "datasets", subComponent: name } }) return constants.METRIC_ALIAS_DELETED_SUCCESSFULLY; } catch (error: any) { throw new Error(constants.METRIC_ALIAS_NOT_DELETED); From a1e8a07cd2b3d18a88a7ecd1546a22c0fe242c44 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 18 Sep 2024 15:28:15 +0530 Subject: [PATCH 234/235] #OBS-I79: feat: test case fixes --- .../DatasetStatusTransition/DatasetRetire.spec.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts index 1a0428af..e09dcc41 100644 --- a/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts +++ b/api-service/src/tests/DatasetManagement/DatasetStatusTransition/DatasetRetire.spec.ts @@ -14,6 +14,7 @@ import { Datasource } from "../../../models/Datasource"; import { commandHttpService } from "../../../connections/commandServiceConnection"; import { druidHttpService } from "../../../connections/druidConnection"; import { sequelize } from "../../../connections/databaseConnection"; +import { datasetService } from "../../../services/DatasetService"; chai.use(spies); chai.should(); @@ -46,6 +47,9 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Datasource, "findAll", () => { return Promise.resolve([{ datasource_ref: "telemetry" }]) }) + chai.spy.on(datasetService, "deleteAlerts", () => { + return Promise.resolve({}) + }) chai.spy.on(druidHttpService, "post", () => { return Promise.resolve({}) }) @@ -91,6 +95,9 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Dataset, "update", () => { return Promise.resolve({}) }) + chai.spy.on(datasetService, "deleteAlerts", () => { + return Promise.resolve({}) + }) chai.spy.on(commandHttpService, "post", () => { return Promise.resolve({}) }) @@ -130,6 +137,9 @@ describe("DATASET STATUS TRANSITION RETIRE", () => { chai.spy.on(Datasource, "update", () => { return Promise.resolve({}) }) + chai.spy.on(datasetService, "deleteAlerts", () => { + return Promise.resolve({}) + }) chai.spy.on(Dataset, "update", () => { return Promise.resolve({}) }) From 1989e53a366771bf6ac0d787f1b2e60c07e26218 Mon Sep 17 00:00:00 2001 From: JeraldJF Date: Wed, 18 Sep 2024 18:21:37 +0530 Subject: [PATCH 235/235] #OBS-I79: feat: Alerts get fix --- api-service/src/controllers/Alerts/Alerts.ts | 5 +++-- api-service/src/services/managers/index.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/api-service/src/controllers/Alerts/Alerts.ts b/api-service/src/controllers/Alerts/Alerts.ts index b3bda19e..76bd9c3a 100644 --- a/api-service/src/controllers/Alerts/Alerts.ts +++ b/api-service/src/controllers/Alerts/Alerts.ts @@ -31,8 +31,9 @@ const createAlertHandler = async (req: Request, res: Response, next: NextFunctio const publishAlertHandler = async (req: Request, res: Response, next: NextFunction) => { try { const { alertId } = req.params; - const rulePayload: Record | null = await getAlertRule(alertId); - if (!rulePayload) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const ruleModel: Record | null = await getAlertRule(alertId); + if (!ruleModel) return next({ message: httpStatus[httpStatus.NOT_FOUND], statusCode: httpStatus.NOT_FOUND }); + const rulePayload = ruleModel.toJSON(); const userID = (req as any)?.userID; _.set(rulePayload, "updated_by", userID); if (rulePayload.status == "live") { diff --git a/api-service/src/services/managers/index.ts b/api-service/src/services/managers/index.ts index 43e796dc..585b32b7 100644 --- a/api-service/src/services/managers/index.ts +++ b/api-service/src/services/managers/index.ts @@ -7,7 +7,7 @@ import { Metrics } from "../../models/Metric"; import constants from "./constants"; export const getAlertRule = (id: string) => { - return Alert.findOne({ where: { id }, raw: true }); + return Alert.findOne({ where: { id } }); } const getService = (manager: string) => {