Merge branch 'main' into issue-#15188

bootstrap/sql/migrations/native/1.4.0/mysql/postDataMigrationSQLScript.sql

@@ -0,0 +1,13 @@
+-- Update the relation between testDefinition and testCase to 0 (CONTAINS)
+UPDATE entity_relationship
+SET relation = 0
+WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0;
+
+-- Update the test definition provider
+-- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user
+UPDATE test_definition
+SET json = CASE
+		WHEN JSON_CONTAINS(json, '"OpenMetadata"', '$.testPlatforms') THEN JSON_INSERT(json,'$.provider','system')
+		ELSE JSON_INSERT(json,'$.provider','user')
+	END
+;

bootstrap/sql/migrations/native/1.4.0/postgres/postDataMigrationSQLScript.sql

@@ -0,0 +1,13 @@
+-- Update the relation between testDefinition and testCase to 0 (CONTAINS)
+UPDATE entity_relationship
+SET relation = 0
+WHERE fromEntity = 'testDefinition' AND toEntity = 'testCase' AND relation != 0;
+
+-- Update the test definition provider
+-- If the test definition has OpenMetadata as a test platform, then the provider is system, else it is user
+UPDATE test_definition
+SET json =
+	case
+		when json->'testPlatforms' @> '"OpenMetadata"' then jsonb_set(json,'{provider}','"system"',true)
+		else jsonb_set(json,'{provider}','"user"', true)
+	end;

ingestion/src/metadata/ingestion/source/dashboard/tableau/metadata.py

@@ -152,6 +152,23 @@ def yield_tags(
                 include_tags=self.source_config.includeTags,
             )
 
+    def _get_datamodel_sql_query(self, data_model: DataSource) -> Optional[str]:
+        """
+        Method to fetch the custom sql query from the tableau datamodels
+        """
+        try:
+            sql_queries = []
+            for table in data_model.upstreamTables or []:
+                for referenced_query in table.referencedByQueries or []:
+                    sql_queries.append(referenced_query.query)
+            return "\n\n".join(sql_queries) or None
+        except Exception as exc:
+            logger.debug(traceback.format_exc())
+            logger.warning(
+                f"Error processing queries for datamodel [{data_model.id}]: {exc}"
+            )
+        return None
+
     def yield_datamodel(
         self, dashboard_details: TableauDashboard
     ) -> Iterable[Either[CreateDashboardDataModelRequest]]:
@@ -171,6 +188,7 @@ def yield_datamodel(
                         dataModelType=DataModelType.TableauDataModel.value,
                         serviceType=DashboardServiceType.Tableau.value,
                         columns=self.get_column_info(data_model),
+                        sql=self._get_datamodel_sql_query(data_model=data_model),
                     )
                     yield Either(right=data_model_request)
                     self.register_record_datamodel(datamodel_request=data_model_request)

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java

@@ -193,7 +193,7 @@ private List<TestSuite> getTestSuites(TestCase test) {
   }
 
   private EntityReference getTestDefinition(TestCase test) {
-    return getFromEntityRef(test.getId(), Relationship.APPLIED_TO, TEST_DEFINITION, true);
+    return getFromEntityRef(test.getId(), Relationship.CONTAINS, TEST_DEFINITION, true);
   }
 
   private void validateTestParameters(
@@ -247,7 +247,7 @@ public void storeRelationships(TestCase test) {
         test.getId(),
         TEST_DEFINITION,
         TEST_CASE,
-        Relationship.APPLIED_TO);
+        Relationship.CONTAINS);
   }
 
   @Override
@@ -900,7 +900,7 @@ public void entitySpecificUpdate() {
           TEST_DEFINITION,
           original.getTestDefinition(),
           updated.getTestDefinition(),
-          Relationship.APPLIED_TO,
+          Relationship.CONTAINS,
           TEST_CASE,
           updated.getId());
       recordChange("parameterValues", original.getParameterValues(), updated.getParameterValues());

openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestDefinitionResource.java

@@ -389,10 +389,15 @@ public Response delete(
           @QueryParam("hardDelete")
           @DefaultValue("false")
           boolean hardDelete,
+      @Parameter(
+              description = "Recursively delete this entity and it's children. (Default `false`)")
+          @QueryParam("recursive")
+          @DefaultValue("false")
+          boolean recursive,
       @Parameter(description = "Id of the test definition", schema = @Schema(type = "UUID"))
           @PathParam("id")
           UUID id) {
-    return delete(uriInfo, securityContext, id, false, hardDelete);
+    return delete(uriInfo, securityContext, id, recursive, hardDelete);
   }
 
   @DELETE
@@ -414,10 +419,15 @@ public Response delete(
           @QueryParam("hardDelete")
           @DefaultValue("false")
           boolean hardDelete,
+      @Parameter(
+              description = "Recursively delete this entity and it's children. (Default `false`)")
+          @QueryParam("recursive")
+          @DefaultValue("false")
+          boolean recursive,
       @Parameter(description = "Name of the test definition", schema = @Schema(type = "string"))
           @PathParam("name")
           String name) {
-    return deleteByName(uriInfo, securityContext, name, false, hardDelete);
+    return deleteByName(uriInfo, securityContext, name, recursive, hardDelete);
   }
 
   @PUT

openmetadata-service/src/main/resources/json/data/tests/columnValueMaxToBeBetween.json

@@ -19,6 +19,7 @@
       "description": "Expected maximum value in the column to be lower or equal than",
       "dataType": "INT"
     }
-  ]
+  ],
+  "provider": "system"
 }
 

openmetadata-service/src/main/resources/json/data/tests/columnValueMeanToBeBetween.json

@@ -19,6 +19,7 @@
       "description": "Expected mean value for the column to be lower or equal than",
       "dataType": "INT"
     }
-  ]
+  ],
+  "provider": "system"
 }
 

openmetadata-service/src/main/resources/json/data/tests/columnValueMedianToBeBetween.json

@@ -19,6 +19,7 @@
         "description": "Expected median value for the column to be lower or equal than",
         "dataType": "INT"
       }
-  ]
+  ],
+  "provider": "system"
 }
 

openmetadata-service/src/main/resources/json/data/tests/columnValueMinToBeBetween.json

@@ -19,6 +19,7 @@
         "description": "Expect minimum value in the column to be lower or equal than",
         "dataType": "INT"
       }
-    ]
+    ],
+    "provider": "system"
   }
 

openmetadata-service/src/main/resources/json/data/tests/columnValueStdDevToBeBetween.json

@@ -19,6 +19,7 @@
         "description": "Expected std. dev value for the column to be lower or equal than",
         "dataType": "INT"
       }
-    ]
+    ],
+    "provider": "system"
   }
 

openmetadata-service/src/main/resources/json/data/tests/columnValuesLengthsToBeBetween.json

@@ -20,5 +20,6 @@
       "dataType": "INT"
     }
   ],
-  "supportsRowLevelPassedFailed": true
+  "supportsRowLevelPassedFailed": true,
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesMissingCountToBeEqual.json

@@ -20,5 +20,6 @@
       "description": "By default match all null and empty values to be missing. This field allows us to configure additional strings such as N/A, NULL as missing strings as well.",
       "dataType": "STRING"
     }
-  ]
+  ],
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesSumToBeBetween.json

@@ -19,6 +19,7 @@
         "description": "Expected sum values in the column to be lower or equal than",
         "dataType": "INT"
       }
-    ]
+    ],
+    "provider": "system"
   }
 

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeBetween.json

@@ -20,5 +20,6 @@
       "dataType": "INT"
     }
   ],
-  "supportsRowLevelPassedFailed": true
+  "supportsRowLevelPassedFailed": true,
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeInSet.json

@@ -15,5 +15,6 @@
       "required": true
     }
   ],
-  "supportsRowLevelPassedFailed": true
+  "supportsRowLevelPassedFailed": true,
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotInSet.json

@@ -15,5 +15,6 @@
       "required": true
     }
   ],
-  "supportsRowLevelPassedFailed": true
+  "supportsRowLevelPassedFailed": true,
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeNotNull.json

@@ -6,5 +6,6 @@
   "entityType": "COLUMN",
   "testPlatforms": ["OpenMetadata"],
   "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
-  "supportsRowLevelPassedFailed": true
+  "supportsRowLevelPassedFailed": true,
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToBeUnique.json

@@ -6,5 +6,6 @@
   "entityType": "COLUMN",
   "testPlatforms": ["OpenMetadata"],
   "supportedDataTypes": ["NUMBER","TINYINT","SMALLINT","INT","BIGINT","BYTEINT","BYTES","FLOAT","DOUBLE","DECIMAL","NUMERIC","TIMESTAMP","TIMESTAMPZ","TIME","DATE","DATETIME","INTERVAL","STRING","MEDIUMTEXT","TEXT","CHAR","VARCHAR","BOOLEAN","BINARY","VARBINARY","ARRAY","BLOB","LONGBLOB","MEDIUMBLOB","MAP","STRUCT","UNION","SET","GEOGRAPHY","ENUM","JSON","UUID","VARIANT","GEOMETRY","POINT","POLYGON"],
-  "supportsRowLevelPassedFailed": true
+  "supportsRowLevelPassedFailed": true,
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToMatchRegex.json

@@ -15,5 +15,6 @@
       "required": true
     }
   ],
-  "supportsRowLevelPassedFailed": true
+  "supportsRowLevelPassedFailed": true,
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/columnValuesToNotMatchRegex.json

@@ -15,5 +15,6 @@
       "required": true
     }
   ],
-  "supportsRowLevelPassedFailed": true
+  "supportsRowLevelPassedFailed": true,
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToBeBetween.json

@@ -18,5 +18,6 @@
       "description": "Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum number of column",
       "dataType": "INT"
     }
-  ]
+  ],
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/tableColumnCountToEqual.json

@@ -13,5 +13,6 @@
       "dataType": "INT",
       "required": true
     }
-  ]
+  ],
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/tableColumnNameToExist.json

@@ -13,6 +13,7 @@
         "dataType": "STRING",
         "required": true
       }
-    ]
+    ],
+    "provider": "system"
   }
 

openmetadata-service/src/main/resources/json/data/tests/tableColumnToMatchSet.json

@@ -19,6 +19,7 @@
         "description": "Whether or not to considered the order of the list when performing the match check",
         "dataType": "BOOLEAN"
       }
-    ]
+    ],
+    "provider": "system"
   }
 

openmetadata-service/src/main/resources/json/data/tests/tableCustomSQLQuery.json

@@ -28,6 +28,7 @@
         "dataType": "NUMBER",
         "required": false
       }
-    ]
+    ],
+    "provider": "system"
   }
 

openmetadata-service/src/main/resources/json/data/tests/tableRowCountToBeBetween.json

@@ -18,5 +18,6 @@
       "description": "Expected number of columns should be less than or equal to {maxValue}. If maxValue is not included, minValue is treated as lowerBound and there will be no maximum",
       "dataType": "INT"
     }
-  ]
+  ],
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/tableRowCountToEqual.json

@@ -13,5 +13,6 @@
       "dataType": "INT",
       "required": true
     }
-  ]
+  ],
+  "provider": "system"
 }

openmetadata-service/src/main/resources/json/data/tests/tableRowInsertedCountToBeBetween.json

@@ -41,6 +41,7 @@
         "dataType": "INT",
         "required": true
       }
-    ]
+    ],
+    "provider": "system"
   }
 

openmetadata-spec/src/main/resources/json/schema/api/tests/createTestDefinition.json

@@ -38,6 +38,9 @@
         "$ref": "../../entity/data/table.json#/definitions/dataType"
       }
     },
+    "provider" : {
+      "$ref": "../../type/basic.json#/definitions/providerType"
+    },
     "parameterDefinition": {
       "type": "array",
       "items": {

openmetadata-spec/src/main/resources/json/schema/tests/testDefinition.json

@@ -131,6 +131,9 @@
       "$ref": "../type/entityReference.json",
       "default": null
     },
+    "provider" : {
+      "$ref": "../type/basic.json#/definitions/providerType"
+    },
     "version": {
       "description": "Metadata version of the entity.",
       "$ref": "../type/entityHistory.json#/definitions/entityVersion"

openmetadata-ui/src/main/resources/ui/src/App.tsx

@@ -17,19 +17,19 @@ import { I18nextProvider } from 'react-i18next';
 import { Router } from 'react-router-dom';
 import { ToastContainer } from 'react-toastify';
 import 'react-toastify/dist/ReactToastify.min.css';
-import ApplicationConfigProvider from './components/ApplicationConfigProvider/ApplicationConfigProvider';
 import AppRouter from './components/AppRouter/AppRouter';
 import { AuthProvider } from './components/Auth/AuthProviders/AuthProvider';
-import DirectionProvider from './components/DirectionProvider/DirectionProvider';
+import ErrorBoundary from './components/common/ErrorBoundary/ErrorBoundary';
 import DomainProvider from './components/Domain/DomainProvider/DomainProvider';
 import { EntityExportModalProvider } from './components/Entity/EntityExportModalProvider/EntityExportModalProvider.component';
-import ErrorBoundary from './components/ErrorBoundary/ErrorBoundary';
-import GlobalSearchProvider from './components/GlobalSearchProvider/GlobalSearchProvider';
-import PermissionProvider from './components/PermissionProvider/PermissionProvider';
-import TourProvider from './components/TourProvider/TourProvider';
 import WebAnalyticsProvider from './components/WebAnalytics/WebAnalyticsProvider';
-import WebSocketProvider from './components/WebSocketProvider/WebSocketProvider';
 import { TOAST_OPTIONS } from './constants/Toasts.constants';
+import ApplicationConfigProvider from './context/ApplicationConfigProvider/ApplicationConfigProvider';
+import DirectionProvider from './context/DirectionProvider/DirectionProvider';
+import GlobalSearchProvider from './context/GlobalSearchProvider/GlobalSearchProvider';
+import PermissionProvider from './context/PermissionProvider/PermissionProvider';
+import TourProvider from './context/TourProvider/TourProvider';
+import WebSocketProvider from './context/WebSocketProvider/WebSocketProvider';
 import { history } from './utils/HistoryUtils';
 import i18n from './utils/i18next/LocalUtil';
 

openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/ActivityFeedCardV1.tsx

@@ -17,14 +17,14 @@ import { noop } from 'lodash';
 import React, { useState } from 'react';
 import { ReactComponent as ThreadIcon } from '../../../assets/svg/thread.svg';
 import UserPopOverCard from '../../../components/common/PopOverCard/UserPopOverCard';
-import Reactions from '../../../components/Reactions/Reactions';
 import { ReactionOperation } from '../../../enums/reactions.enum';
 import {
   Post,
   ReactionType,
   Thread,
 } from '../../../generated/entity/feed/thread';
 import { useActivityFeedProvider } from '../ActivityFeedProvider/ActivityFeedProvider';
+import Reactions from '../Reactions/Reactions';
 import ActivityFeedActions from '../Shared/ActivityFeedActions';
 import './activity-feed-card.style.less';
 import FeedCardBodyV1 from './FeedCardBody/FeedCardBodyV1';

openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBody.tsx

@@ -22,8 +22,8 @@ import {
   MarkdownToHTMLConverter,
 } from '../../../../utils/FeedUtils';
 import RichTextEditorPreviewer from '../../../common/RichTextEditor/RichTextEditorPreviewer';
-import Reactions from '../../../Reactions/Reactions';
 import ActivityFeedEditor from '../../ActivityFeedEditor/ActivityFeedEditor';
+import Reactions from '../../Reactions/Reactions';
 import { FeedBodyProp } from '../ActivityFeedCard.interface';
 
 const FeedCardBody: FC<FeedBodyProp> = ({