Merge branch 'main' of https://github.com/open-metadata/OpenMetadata …

…into issue_15742

bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql

@@ -0,0 +1,5 @@
+-- Make domain policy and role non-system
+UPDATE policy_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessPolicy';
+UPDATE policy_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessPolicy';
+UPDATE role_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessRole';
+UPDATE role_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessRole';

bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql

@@ -0,0 +1,5 @@
+-- Make domain policy and role non-system
+UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessPolicy';
+UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessPolicy';
+UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessRole';
+UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessRole';

conf/openmetadata.yaml

@@ -180,6 +180,7 @@ authenticationConfiguration:
   # This will only be valid when provider type specified is customOidc
   providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
   publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
+  tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"}
   authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
   clientId: ${AUTHENTICATION_CLIENT_ID:-""}
   callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}

openmetadata-docs/content/v1.6.x/connectors/dashboard/powerbi-report-server/index.md

@@ -1,6 +1,6 @@
 ---
 title: Run the PowerBI Report Server Connector Externally
-slug: /connectors/dashboard/powerbireportserver/yaml
+slug: /connectors/dashboard/powerbireportserver
 ---
 
 {% connectorDetailsHeader

openmetadata-docs/content/v1.7.x-SNAPSHOT/connectors/dashboard/powerbi-report-server/index.md

@@ -1,6 +1,6 @@
 ---
 title: Run the PowerBI Report Server Connector Externally
-slug: /connectors/dashboard/powerbireportserver/yaml
+slug: /connectors/dashboard/powerbireportserver
 ---
 
 {% connectorDetailsHeader

openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java

@@ -79,6 +79,7 @@
 import org.openmetadata.service.exception.JsonMappingExceptionMapper;
 import org.openmetadata.service.exception.OMErrorPageHandler;
 import org.openmetadata.service.fernet.Fernet;
+import org.openmetadata.service.governance.workflows.WorkflowHandler;
 import org.openmetadata.service.jdbi3.CollectionDAO;
 import org.openmetadata.service.jdbi3.EntityRepository;
 import org.openmetadata.service.jdbi3.MigrationDAO;
@@ -173,6 +174,9 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ
     // Configure the Fernet instance
     Fernet.getInstance().setFernetKey(catalogConfig);
 
+    // Initialize Workflow Handler
+    WorkflowHandler.initialize(catalogConfig);
+
     // Init Settings Cache after repositories
     SettingsCache.initialize(catalogConfig);
 
@@ -186,7 +190,10 @@ public void run(OpenMetadataApplicationConfig catalogConfig, Environment environ
     EntityMaskerFactory.createEntityMasker();
 
     // Instantiate JWT Token Generator
-    JWTTokenGenerator.getInstance().init(catalogConfig.getJwtTokenConfiguration());
+    JWTTokenGenerator.getInstance()
+        .init(
+            catalogConfig.getAuthenticationConfiguration().getTokenValidationAlgorithm(),
+            catalogConfig.getJwtTokenConfiguration());
 
     // Set the Database type for choosing correct queries from annotations
     jdbi.getConfig(SqlObjects.class)

openmetadata-service/src/main/java/org/openmetadata/service/governance/workflows/WorkflowHandler.java

@@ -64,7 +64,7 @@ public void initializeNewProcessEngine(
       ProcessEngineConfiguration currentProcessEngineConfiguration) {
     ProcessEngines.destroy();
     SystemRepository systemRepository = Entity.getSystemRepository();
-    WorkflowSettings workflowSettings = systemRepository.getWorkflowSettings();
+    WorkflowSettings workflowSettings = systemRepository.getWorkflowSettingsOrDefault();
 
     StandaloneProcessEngineConfiguration processEngineConfiguration =
         new StandaloneProcessEngineConfiguration();

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java

@@ -131,6 +131,7 @@
 import org.openmetadata.schema.entity.feed.Suggestion;
 import org.openmetadata.schema.entity.teams.Team;
 import org.openmetadata.schema.entity.teams.User;
+import org.openmetadata.schema.entity.type.Style;
 import org.openmetadata.schema.system.EntityError;
 import org.openmetadata.schema.type.ApiStatus;
 import org.openmetadata.schema.type.AssetCertification;
@@ -2969,6 +2970,14 @@ private static List<EntityReference> getEntityReferences(List<EntityReference> r
 
     private void updateStyle() {
       if (supportsStyle) {
+        Style originalStyle = original.getStyle();
+        Style updatedStyle = updated.getStyle();
+
+        if (originalStyle == updatedStyle) return;
+        if (operation == Operation.PUT && updatedStyle == null) {
+          updatedStyle = originalStyle;
+          updated.setStyle(updatedStyle);
+        }
         recordChange(FIELD_STYLE, original.getStyle(), updated.getStyle(), true);
       }
     }
@@ -3023,7 +3032,7 @@ private void updateCertification() {
 
       SystemRepository systemRepository = Entity.getSystemRepository();
       AssetCertificationSettings assetCertificationSettings =
-          systemRepository.getAssetCertificationSettings();
+          systemRepository.getAssetCertificationSettingOrDefault();
 
       String certificationLabel = updatedCertification.getTagLabel().getTagFQN();
 

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java

@@ -16,6 +16,8 @@
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.api.configuration.UiThemePreference;
 import org.openmetadata.schema.configuration.AssetCertificationSettings;
+import org.openmetadata.schema.configuration.ExecutorConfiguration;
+import org.openmetadata.schema.configuration.HistoryCleanUpConfiguration;
 import org.openmetadata.schema.configuration.WorkflowSettings;
 import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse;
@@ -121,6 +123,17 @@ public AssetCertificationSettings getAssetCertificationSettings() {
         .orElse(null);
   }
 
+  public AssetCertificationSettings getAssetCertificationSettingOrDefault() {
+    AssetCertificationSettings assetCertificationSettings = getAssetCertificationSettings();
+    if (assetCertificationSettings == null) {
+      assetCertificationSettings =
+          new AssetCertificationSettings()
+              .withAllowedClassification("Certification")
+              .withValidityPeriod("P30D");
+    }
+    return assetCertificationSettings;
+  }
+
   public WorkflowSettings getWorkflowSettings() {
     Optional<Settings> oWorkflowSettings =
         Optional.ofNullable(getConfigWithKey(SettingsType.WORKFLOW_SETTINGS.value()));
@@ -130,6 +143,17 @@ public WorkflowSettings getWorkflowSettings() {
         .orElse(null);
   }
 
+  public WorkflowSettings getWorkflowSettingsOrDefault() {
+    WorkflowSettings workflowSettings = getWorkflowSettings();
+    if (workflowSettings == null) {
+      workflowSettings =
+          new WorkflowSettings()
+              .withExecutorConfiguration(new ExecutorConfiguration())
+              .withHistoryCleanUpConfiguration(new HistoryCleanUpConfiguration());
+    }
+    return workflowSettings;
+  }
+
   public Settings getEmailConfigInternal() {
     try {
       Settings setting = dao.getConfigWithKey(SettingsType.EMAIL_CONFIGURATION.value());

openmetadata-service/src/main/java/org/openmetadata/service/resources/governance/WorkflowDefinitionResource.java

@@ -70,7 +70,6 @@ public static class WorkflowDefinitionList extends ResultList<WorkflowDefinition
 
   @Override
   public void initialize(OpenMetadataApplicationConfig config) throws IOException {
-    WorkflowHandler.initialize(config);
     repository.initSeedDataFromResources();
   }
 

openmetadata-service/src/main/java/org/openmetadata/service/secrets/converter/ClassConverterFactory.java

@@ -13,10 +13,8 @@
 
 package org.openmetadata.service.secrets.converter;
 
-import com.mysql.cj.MysqlConnection;
 import java.util.Map;
 import lombok.Getter;
-import org.flywaydb.core.internal.database.redshift.RedshiftConnection;
 import org.openmetadata.schema.auth.SSOAuthMechanism;
 import org.openmetadata.schema.entity.automations.TestServiceConnectionRequest;
 import org.openmetadata.schema.entity.automations.Workflow;
@@ -34,7 +32,9 @@
 import org.openmetadata.schema.services.connections.database.GreenplumConnection;
 import org.openmetadata.schema.services.connections.database.HiveConnection;
 import org.openmetadata.schema.services.connections.database.IcebergConnection;
+import org.openmetadata.schema.services.connections.database.MysqlConnection;
 import org.openmetadata.schema.services.connections.database.PostgresConnection;
+import org.openmetadata.schema.services.connections.database.RedshiftConnection;
 import org.openmetadata.schema.services.connections.database.SalesforceConnection;
 import org.openmetadata.schema.services.connections.database.SapHanaConnection;
 import org.openmetadata.schema.services.connections.database.TrinoConnection;

openmetadata-service/src/main/java/org/openmetadata/service/security/JwtFilter.java

@@ -22,6 +22,7 @@
 import static org.openmetadata.service.security.SecurityUtil.validatePrincipalClaimsMapping;
 import static org.openmetadata.service.security.jwt.JWTTokenGenerator.ROLES_CLAIM;
 import static org.openmetadata.service.security.jwt.JWTTokenGenerator.TOKEN_TYPE;
+import static org.openmetadata.service.security.jwt.JWTTokenGenerator.getAlgorithm;
 
 import com.auth0.jwk.Jwk;
 import com.auth0.jwk.JwkProvider;
@@ -71,6 +72,7 @@ public class JwtFilter implements ContainerRequestFilter {
   private boolean enforcePrincipalDomain;
   private AuthProvider providerType;
   private boolean useRolesFromProvider = false;
+  private AuthenticationConfiguration.TokenValidationAlgorithm tokenValidationAlgorithm;
 
   private static final List<String> DEFAULT_PUBLIC_KEY_URLS =
       Arrays.asList(
@@ -123,6 +125,7 @@ public JwtFilter(
     this.principalDomain = authorizerConfiguration.getPrincipalDomain();
     this.enforcePrincipalDomain = authorizerConfiguration.getEnforcePrincipalDomain();
     this.useRolesFromProvider = authorizerConfiguration.getUseRolesFromProvider();
+    this.tokenValidationAlgorithm = authenticationConfiguration.getTokenValidationAlgorithm();
   }
 
   @VisibleForTesting
@@ -224,7 +227,8 @@ public Map<String, Claim> validateJwtAndGetClaims(String token) {
 
     // Validate JWT with public key
     Jwk jwk = jwkProvider.get(jwt.getKeyId());
-    Algorithm algorithm = Algorithm.RSA256((RSAPublicKey) jwk.getPublicKey(), null);
+    Algorithm algorithm =
+        getAlgorithm(tokenValidationAlgorithm, (RSAPublicKey) jwk.getPublicKey(), null);
     try {
       algorithm.verify(jwt);
     } catch (RuntimeException runtimeException) {

openmetadata-service/src/main/java/org/openmetadata/service/security/jwt/JWTTokenGenerator.java

@@ -37,6 +37,7 @@
 import java.util.Set;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.schema.api.security.AuthenticationConfiguration;
 import org.openmetadata.schema.api.security.jwt.JWTTokenConfiguration;
 import org.openmetadata.schema.auth.JWTAuthMechanism;
 import org.openmetadata.schema.auth.JWTTokenExpiry;
@@ -56,6 +57,7 @@ public class JWTTokenGenerator {
   @Getter private RSAPublicKey publicKey;
   private String issuer;
   private String kid;
+  private AuthenticationConfiguration.TokenValidationAlgorithm tokenValidationAlgorithm;
 
   private JWTTokenGenerator() {
     /* Private constructor for singleton */
@@ -66,7 +68,9 @@ public static JWTTokenGenerator getInstance() {
   }
 
   /** Expected to be initialized only once during application start */
-  public void init(JWTTokenConfiguration jwtTokenConfiguration) {
+  public void init(
+      AuthenticationConfiguration.TokenValidationAlgorithm algorithm,
+      JWTTokenConfiguration jwtTokenConfiguration) {
     try {
       if (jwtTokenConfiguration.getRsaprivateKeyFilePath() != null
           && !jwtTokenConfiguration.getRsaprivateKeyFilePath().isEmpty()
@@ -84,6 +88,7 @@ public void init(JWTTokenConfiguration jwtTokenConfiguration) {
         publicKey = (RSAPublicKey) kf.generatePublic(spec);
         issuer = jwtTokenConfiguration.getJwtissuer();
         kid = jwtTokenConfiguration.getKeyId();
+        tokenValidationAlgorithm = algorithm;
       }
     } catch (Exception ex) {
       LOG.error("Failed to initialize JWTTokenGenerator ", ex);
@@ -141,7 +146,7 @@ public JWTAuthMechanism getJwtAuthMechanism(
         }
       }
       JWTAuthMechanism jwtAuthMechanism = new JWTAuthMechanism().withJWTTokenExpiry(expiry);
-      Algorithm algorithm = Algorithm.RSA256(null, privateKey);
+      Algorithm algorithm = getAlgorithm(tokenValidationAlgorithm, null, privateKey);
       String token =
           JWT.create()
               .withIssuer(issuer)
@@ -214,4 +219,15 @@ public Date getTokenExpiryFromJWT(String token) {
 
     return jwt.getExpiresAt();
   }
+
+  public static Algorithm getAlgorithm(
+      AuthenticationConfiguration.TokenValidationAlgorithm algorithm,
+      RSAPublicKey publicKey,
+      RSAPrivateKey privateKey) {
+    return switch (algorithm) {
+      case RS_256 -> Algorithm.RSA256(publicKey, privateKey);
+      case RS_384 -> Algorithm.RSA384(publicKey, privateKey);
+      case RS_512 -> Algorithm.RSA512(publicKey, privateKey);
+    };
+  }
 }

openmetadata-service/src/main/resources/json/data/policy/DomainAccessPolicy.json

@@ -4,8 +4,8 @@
   "fullyQualifiedName": "DomainOnlyAccessPolicy",
   "description": "This Policy adds restrictions so that users will have access to domain related data. If the user has some domain, then he will be able to access data only for that domain. If the user does not have any domain assigned , he will be able to access only assets which also does not have any domain.",
   "enabled": true,
-  "allowDelete": false,
-  "provider": "system",
+  "allowDelete": true,
+  "provider": "user",
   "rules": [
     {
       "name": "DomainOnlyAccessRule",

openmetadata-service/src/main/resources/json/data/role/DomainOnlyAccessRole.json

@@ -2,8 +2,8 @@
   "name": "DomainOnlyAccessRole",
   "displayName": "Domain Only Access Role",
   "description": "Role Corresponding to Domain Access Restriction.",
-  "allowDelete": false,
-  "provider": "system",
+  "allowDelete": true,
+  "provider": "user",
   "policies" : [
     {
       "type" : "policy",

openmetadata-service/src/test/java/org/openmetadata/service/resources/system/SystemResourceTest.java

@@ -47,6 +47,7 @@
 import org.openmetadata.schema.auth.JWTAuthMechanism;
 import org.openmetadata.schema.auth.JWTTokenExpiry;
 import org.openmetadata.schema.configuration.AssetCertificationSettings;
+import org.openmetadata.schema.configuration.WorkflowSettings;
 import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.entity.teams.AuthenticationMechanism;
@@ -551,6 +552,48 @@ void testLineageSettings() throws HttpResponseException {
     assertEquals(4, updatedLineageConfig.getDownstreamDepth());
   }
 
+  @Test
+  void testWorkflowSettings() throws HttpResponseException {
+    // Retrieve the default workflow settings
+    Settings setting = getSystemConfig(SettingsType.WORKFLOW_SETTINGS);
+    WorkflowSettings workflowSettings =
+        JsonUtils.convertValue(setting.getConfigValue(), WorkflowSettings.class);
+
+    // Assert default values
+    assertEquals(50, workflowSettings.getExecutorConfiguration().getCorePoolSize());
+    assertEquals(1000, workflowSettings.getExecutorConfiguration().getQueueSize());
+    assertEquals(100, workflowSettings.getExecutorConfiguration().getMaxPoolSize());
+    assertEquals(20, workflowSettings.getExecutorConfiguration().getTasksDuePerAcquisition());
+    assertEquals(7, workflowSettings.getHistoryCleanUpConfiguration().getCleanAfterNumberOfDays());
+
+    // Update workflow settings
+    workflowSettings.getExecutorConfiguration().setCorePoolSize(100);
+    workflowSettings.getExecutorConfiguration().setQueueSize(2000);
+    workflowSettings.getExecutorConfiguration().setMaxPoolSize(200);
+    workflowSettings.getExecutorConfiguration().setTasksDuePerAcquisition(40);
+    workflowSettings.getHistoryCleanUpConfiguration().setCleanAfterNumberOfDays(10);
+
+    Settings updatedSetting =
+        new Settings()
+            .withConfigType(SettingsType.WORKFLOW_SETTINGS)
+            .withConfigValue(workflowSettings);
+
+    updateSystemConfig(updatedSetting);
+
+    // Retrieve the updated settings
+    Settings updatedSettings = getSystemConfig(SettingsType.WORKFLOW_SETTINGS);
+    WorkflowSettings updateWorkflowSettings =
+        JsonUtils.convertValue(updatedSettings.getConfigValue(), WorkflowSettings.class);
+
+    // Assert updated values
+    assertEquals(100, updateWorkflowSettings.getExecutorConfiguration().getCorePoolSize());
+    assertEquals(2000, updateWorkflowSettings.getExecutorConfiguration().getQueueSize());
+    assertEquals(200, updateWorkflowSettings.getExecutorConfiguration().getMaxPoolSize());
+    assertEquals(40, updateWorkflowSettings.getExecutorConfiguration().getTasksDuePerAcquisition());
+    assertEquals(
+        10, updateWorkflowSettings.getHistoryCleanUpConfiguration().getCleanAfterNumberOfDays());
+  }
+
   @Test
   void globalProfilerConfig(TestInfo test) throws HttpResponseException {
     // Create a profiler config

openmetadata-service/src/test/java/org/openmetadata/service/security/JWTTokenGeneratorTest.java

@@ -15,6 +15,7 @@
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
+import org.openmetadata.schema.api.security.AuthenticationConfiguration;
 import org.openmetadata.schema.api.security.jwt.JWTTokenConfiguration;
 import org.openmetadata.schema.auth.JWTAuthMechanism;
 import org.openmetadata.schema.auth.JWTTokenExpiry;
@@ -38,7 +39,8 @@ public void setup() {
     jwtTokenConfiguration.setRsaprivateKeyFilePath(rsaPrivateKeyPath);
     jwtTokenConfiguration.setRsapublicKeyFilePath(rsaPublicKeyPath);
     jwtTokenGenerator = JWTTokenGenerator.getInstance();
-    jwtTokenGenerator.init(jwtTokenConfiguration);
+    jwtTokenGenerator.init(
+        AuthenticationConfiguration.TokenValidationAlgorithm.RS_256, jwtTokenConfiguration);
   }
 
   @Test

openmetadata-spec/src/main/resources/json/schema/configuration/authenticationConfiguration.json

@@ -46,6 +46,12 @@
         "type": "string"
       }
     },
+    "tokenValidationAlgorithm": {
+      "description": "Token Validation Algorithm to use.",
+      "type": "string",
+      "enum": ["RS256", "RS384", "RS512"],
+      "default": "RS256"
+    },
     "authority": {
       "description": "Authentication Authority",
       "type": "string"

openmetadata-spec/src/main/resources/json/schema/entity/services/connections/database/mysqlConnection.json

@@ -44,6 +44,7 @@
     "authType": {
       "title": "Auth Configuration Type",
       "description": "Choose Auth Config Type.",
+      "mask":  true,
       "oneOf": [
         {
           "$ref": "./common/basicAuth.json"