diff --git a/openmetadata-docs/content/v1.5.x/deployment/azure-auth.md b/openmetadata-docs/content/v1.5.x/deployment/azure-auth.md new file mode 100644 index 000000000000..6d84d776110d --- /dev/null +++ b/openmetadata-docs/content/v1.5.x/deployment/azure-auth.md @@ -0,0 +1,29 @@ +--- +title: How to enable Azure Auth +slug: /deployment/azure-auth +collate: false +--- + +# AZURE resources on Postgres/MySQL Auth +https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-extensions#how-to-use-postgresql-extensions +# Requirements + +1. Azure Postgres or MySQL Cluster with auth enabled +2. User on DB Cluster with authentication enabled + +# How to enable Azure Auth on postgresql + +Set the environment variables + +```Commandline + DB_PARAMS="azure=true&allowPublicKeyRetrieval=true&sslmode=require&serverTimezone=UTC" + DB_USER_PASSWORD=none +``` + +Either through helm (if deployed in kubernetes) or as env vars. + +{% note %} + +The `DB_USER_PASSWORD` is still required and cannot be empty. Set it to a random/dummy string. + +{% /note %} diff --git a/openmetadata-docs/content/v1.5.x/menu.md b/openmetadata-docs/content/v1.5.x/menu.md index d8e81edd3eea..0e9732774a2f 100644 --- a/openmetadata-docs/content/v1.5.x/menu.md +++ b/openmetadata-docs/content/v1.5.x/menu.md @@ -185,6 +185,8 @@ site_menu: - category: Deployment / How to enable AWS RDS IAM Auth url: /deployment/rds-iam-auth + - category: Deployment / How to enable Azure Database Auth + url: /deployment/azure-auth - category: Deployment / Server Configuration Reference url: /deployment/configuration - category: Deployment / Database Connection Pooling diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/azure-auth.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/azure-auth.md new file mode 100644 index 000000000000..6d84d776110d --- /dev/null +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/azure-auth.md @@ -0,0 +1,29 @@ +--- +title: How to enable Azure Auth +slug: /deployment/azure-auth +collate: false +--- + +# AZURE resources on Postgres/MySQL Auth +https://learn.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-extensions#how-to-use-postgresql-extensions +# Requirements + +1. Azure Postgres or MySQL Cluster with auth enabled +2. User on DB Cluster with authentication enabled + +# How to enable Azure Auth on postgresql + +Set the environment variables + +```Commandline + DB_PARAMS="azure=true&allowPublicKeyRetrieval=true&sslmode=require&serverTimezone=UTC" + DB_USER_PASSWORD=none +``` + +Either through helm (if deployed in kubernetes) or as env vars. + +{% note %} + +The `DB_USER_PASSWORD` is still required and cannot be empty. Set it to a random/dummy string. + +{% /note %} diff --git a/openmetadata-docs/content/v1.6.x-SNAPSHOT/menu.md b/openmetadata-docs/content/v1.6.x-SNAPSHOT/menu.md index bc7d87029cde..508ecc379560 100644 --- a/openmetadata-docs/content/v1.6.x-SNAPSHOT/menu.md +++ b/openmetadata-docs/content/v1.6.x-SNAPSHOT/menu.md @@ -185,6 +185,8 @@ site_menu: - category: Deployment / How to enable AWS RDS IAM Auth url: /deployment/rds-iam-auth + - category: Deployment / How to enable Azure Auth + url: /deployment/azure-auth - category: Deployment / Server Configuration Reference url: /deployment/configuration - category: Deployment / Database Connection Pooling diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml index 7aea1ae9a4d8..b7994f011579 100644 --- a/openmetadata-service/pom.xml +++ b/openmetadata-service/pom.xml @@ -17,8 +17,9 @@ ${project.basedir}/src/test/java 1.20.1 2.27.17 - 1.13.2 + 1.14.0 4.8.6 + 1.0.0 0.5.11 2.9.0 2.3.4 @@ -290,6 +291,11 @@ azure-identity ${azure-identity.version} + + com.azure + azure-identity-extensions + ${azure-identity-extensions.version} + io.dropwizard.modules dropwizard-web @@ -639,6 +645,10 @@ jakarta.activation 2.0.1 + + com.microsoft.azure + msal4j + diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java index 565a74852707..4fcdbe40123e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/OpenMetadataApplication.java @@ -13,13 +13,13 @@ package org.openmetadata.service; +import static org.openmetadata.service.util.jdbi.JdbiUtils.createAndSetupJDBI; + import io.dropwizard.Application; import io.dropwizard.configuration.EnvironmentVariableSubstitutor; import io.dropwizard.configuration.SubstitutingSourceProvider; -import io.dropwizard.db.DataSourceFactory; import io.dropwizard.health.conf.HealthConfiguration; import io.dropwizard.health.core.HealthCheckBundle; -import io.dropwizard.jdbi3.JdbiFactory; import io.dropwizard.jersey.errors.EarlyEofExceptionMapper; import io.dropwizard.jersey.errors.LoggingExceptionMapper; import io.dropwizard.jersey.jackson.JsonProcessingExceptionMapper; @@ -59,7 +59,6 @@ import org.glassfish.jersey.media.multipart.MultiPartFeature; import org.glassfish.jersey.server.ServerProperties; import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.core.statement.SqlStatements; import org.jdbi.v3.sqlobject.SqlObjects; import org.openmetadata.schema.api.security.AuthenticationConfiguration; import org.openmetadata.schema.api.security.AuthorizerConfiguration; @@ -126,8 +125,6 @@ import org.openmetadata.service.socket.WebSocketManager; import org.openmetadata.service.util.MicrometerBundleSingleton; import org.openmetadata.service.util.incidentSeverityClassifier.IncidentSeverityClassifierInterface; -import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory; -import org.openmetadata.service.util.jdbi.OMSqlLogger; import org.pac4j.core.util.CommonHelper; import org.quartz.SchedulerException; @@ -389,28 +386,6 @@ private void registerSamlServlets( } } - private Jdbi createAndSetupJDBI(Environment environment, DataSourceFactory dbFactory) { - // Check for db auth providers. - DatabaseAuthenticationProviderFactory.get(dbFactory.getUrl()) - .ifPresent( - databaseAuthenticationProvider -> { - String token = - databaseAuthenticationProvider.authenticate( - dbFactory.getUrl(), dbFactory.getUser(), dbFactory.getPassword()); - dbFactory.setPassword(token); - }); - - Jdbi jdbiInstance = new JdbiFactory().build(environment, dbFactory, "database"); - jdbiInstance.setSqlLogger(new OMSqlLogger()); - // Set the Database type for choosing correct queries from annotations - jdbiInstance - .getConfig(SqlObjects.class) - .setSqlLocator(new ConnectionAwareAnnotationSqlLocator(dbFactory.getDriverClass())); - jdbiInstance.getConfig(SqlStatements.class).setUnusedBindingAllowed(true); - - return jdbiInstance; - } - @SneakyThrows @Override public void initialize(Bootstrap bootstrap) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/AzureTokenProvider.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/AzureTokenProvider.java new file mode 100644 index 000000000000..73a2f4929e6e --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/AzureTokenProvider.java @@ -0,0 +1,28 @@ +package org.openmetadata.service.util; + +import com.microsoft.aad.msal4j.*; +import java.net.MalformedURLException; +import java.util.Set; + +public class AzureTokenProvider { + + private static final String CLIENT_ID = "your-client-id"; // From Azure AD App Registration + private static final String TENANT_ID = "your-tenant-id"; // Your Azure AD tenant ID + private static final String CLIENT_SECRET = "your-client-secret"; // Generated in App Registration + private static final String SCOPE = + "https://ossrdbms-aad.database.windows.net/.default"; // Scope for PostgreSQL + + public static String getAccessToken() throws MalformedURLException { + ConfidentialClientApplication app = + ConfidentialClientApplication.builder( + CLIENT_ID, ClientCredentialFactory.createFromSecret(CLIENT_SECRET)) + .authority("https://login.microsoftonline.com/" + TENANT_ID) // Azure AD authority + .build(); + + Set scopes = Set.of(SCOPE); + ClientCredentialParameters parameters = ClientCredentialParameters.builder(scopes).build(); + IAuthenticationResult result = app.acquireToken(parameters).join(); // Get the token + + return result.accessToken(); // Return the access token + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index 16b93c081f8d..5e9c239edf0a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -8,7 +8,6 @@ import ch.qos.logback.classic.Level; import ch.qos.logback.classic.Logger; -import com.codahale.metrics.NoopMetricRegistry; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; import com.google.gson.JsonElement; @@ -37,8 +36,6 @@ import org.flywaydb.core.Flyway; import org.flywaydb.core.api.MigrationVersion; import org.jdbi.v3.core.Jdbi; -import org.jdbi.v3.sqlobject.SqlObjectPlugin; -import org.jdbi.v3.sqlobject.SqlObjects; import org.openmetadata.schema.EntityInterface; import org.openmetadata.schema.ServiceEntityInterface; import org.openmetadata.schema.entity.app.App; @@ -63,7 +60,6 @@ import org.openmetadata.service.jdbi3.ListFilter; import org.openmetadata.service.jdbi3.MigrationDAO; import org.openmetadata.service.jdbi3.SystemRepository; -import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator; import org.openmetadata.service.jdbi3.locator.ConnectionType; import org.openmetadata.service.migration.api.MigrationWorkflow; import org.openmetadata.service.resources.CollectionRegistry; @@ -73,6 +69,7 @@ import org.openmetadata.service.secrets.SecretsManagerFactory; import org.openmetadata.service.secrets.SecretsManagerUpdateService; import org.openmetadata.service.util.jdbi.DatabaseAuthenticationProviderFactory; +import org.openmetadata.service.util.jdbi.JdbiUtils; import org.slf4j.LoggerFactory; import picocli.CommandLine; import picocli.CommandLine.Command; @@ -211,6 +208,7 @@ public Integer dropCreate() { flyway.clean(); LOG.info("Creating the OpenMetadata Schema."); flyway.migrate(); + LOG.info("Running the Native Migrations."); validateAndRunSystemDataMigrations(true); LOG.info("OpenMetadata Database Schema is Updated."); LOG.info("create indexes."); @@ -544,6 +542,9 @@ private void parseConfig() throws Exception { String jdbcUrl = dataSourceFactory.getUrl(); String user = dataSourceFactory.getUser(); String password = dataSourceFactory.getPassword(); + LOG.info("JDBC URL: {}", jdbcUrl); + LOG.info("User: {}", user); + LOG.info("Password: {}", password); assert user != null && password != null; String flywayRootPath = config.getMigrationConfiguration().getFlywayPath(); @@ -568,12 +569,8 @@ private void parseConfig() throws Exception { .load(); nativeSQLScriptRootPath = config.getMigrationConfiguration().getNativePath(); extensionSQLScriptRootPath = config.getMigrationConfiguration().getExtensionPath(); - jdbi = Jdbi.create(dataSourceFactory.build(new NoopMetricRegistry(), "open-metadata-ops")); - jdbi.installPlugin(new SqlObjectPlugin()); - jdbi.getConfig(SqlObjects.class) - .setSqlLocator( - new ConnectionAwareAnnotationSqlLocator( - config.getDataSourceFactory().getDriverClass())); + + jdbi = JdbiUtils.createAndSetupJDBI(dataSourceFactory); searchRepository = new SearchRepository(config.getElasticSearchConfiguration()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/AwsRdsDatabaseAuthenticationProvider.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/AwsRdsDatabaseAuthenticationProvider.java index 9525693497de..ed73d48bfc1c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/AwsRdsDatabaseAuthenticationProvider.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/AwsRdsDatabaseAuthenticationProvider.java @@ -2,13 +2,8 @@ import java.net.MalformedURLException; import java.net.URI; -import java.net.URL; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; -import org.jetbrains.annotations.NotNull; import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.rds.RdsUtilities; @@ -27,9 +22,8 @@ public class AwsRdsDatabaseAuthenticationProvider implements DatabaseAuthenticat @Override public String authenticate(String jdbcUrl, String username, String password) { - // !! try { - // Prepare + URI uri = URI.create(PROTOCOL + removeProtocolFrom(jdbcUrl)); Map queryParams = parseQueryParams(uri.toURL()); @@ -63,27 +57,4 @@ public String authenticate(String jdbcUrl, String username, String password) { throw new DatabaseAuthenticationProviderException(e); } } - - @NotNull - private static String removeProtocolFrom(String jdbcUrl) { - return jdbcUrl.substring(jdbcUrl.indexOf("://") + 3); - } - - private Map parseQueryParams(URL url) { - // Prepare - Map queryPairs = new LinkedHashMap<>(); - String query = url.getQuery(); - String[] pairs = query.split("&"); - - // Loop - for (String pair : pairs) { - int idx = pair.indexOf("="); - // Add - queryPairs.put( - URLDecoder.decode(pair.substring(0, idx), StandardCharsets.UTF_8), - URLDecoder.decode(pair.substring(idx + 1), StandardCharsets.UTF_8)); - } - // Return - return queryPairs; - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/AzureDatabaseAuthenticationProvider.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/AzureDatabaseAuthenticationProvider.java new file mode 100644 index 000000000000..d395efe47bc2 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/AzureDatabaseAuthenticationProvider.java @@ -0,0 +1,36 @@ +package org.openmetadata.service.util.jdbi; + +import com.azure.core.credential.AccessToken; +import com.azure.core.credential.TokenRequestContext; +import com.azure.identity.DefaultAzureCredential; +import com.azure.identity.DefaultAzureCredentialBuilder; + +public class AzureDatabaseAuthenticationProvider implements DatabaseAuthenticationProvider { + public static final String AZURE = "azure"; + + @Override + public String authenticate(String jdbcUrl, String username, String password) { + try { + return fetchAzureADToken(); + } catch (Exception e) { + throw new DatabaseAuthenticationProviderException(e); + } + } + + private String fetchAzureADToken() { + try { + DefaultAzureCredential defaultCredential = new DefaultAzureCredentialBuilder().build(); + TokenRequestContext requestContext = + new TokenRequestContext().addScopes("https://ossrdbms-aad.database.windows.net/.default"); + AccessToken token = defaultCredential.getToken(requestContext).block(); + + if (token != null) { + return token.getToken(); + } else { + throw new DatabaseAuthenticationProviderException("Failed to fetch token"); + } + } catch (Exception e) { + throw new DatabaseAuthenticationProviderException("Error fetching Azure AD token", e); + } + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/DatabaseAuthenticationProvider.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/DatabaseAuthenticationProvider.java index 18f144c298de..49c96c816aa1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/DatabaseAuthenticationProvider.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/DatabaseAuthenticationProvider.java @@ -1,5 +1,12 @@ package org.openmetadata.service.util.jdbi; +import java.net.URL; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.LinkedHashMap; +import java.util.Map; +import javax.validation.constraints.NotNull; + /** * Database authentication provider is the main interface responsible for all implementation that requires additional * authentication steps required by the database in order to authorize a user to be able to operate on it. @@ -15,4 +22,27 @@ public interface DatabaseAuthenticationProvider { * @return authorization token */ String authenticate(String jdbcUrl, String username, String password); + + @NotNull + default String removeProtocolFrom(String jdbcUrl) { + return jdbcUrl.substring(jdbcUrl.indexOf("://") + 3); + } + + default Map parseQueryParams(URL url) { + // Prepare + Map queryPairs = new LinkedHashMap<>(); + String query = url.getQuery(); + String[] pairs = query.split("&"); + + // Loop + for (String pair : pairs) { + int idx = pair.indexOf("="); + // Add + queryPairs.put( + URLDecoder.decode(pair.substring(0, idx), StandardCharsets.UTF_8), + URLDecoder.decode(pair.substring(idx + 1), StandardCharsets.UTF_8)); + } + // Return + return queryPairs; + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/DatabaseAuthenticationProviderFactory.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/DatabaseAuthenticationProviderFactory.java index e5bb6dc7f5a1..030bae257a20 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/DatabaseAuthenticationProviderFactory.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/DatabaseAuthenticationProviderFactory.java @@ -17,8 +17,9 @@ public static Optional get(String jdbcURL) { // Check if (jdbcURL.contains(AwsRdsDatabaseAuthenticationProvider.AWS_REGION) && jdbcURL.contains(AwsRdsDatabaseAuthenticationProvider.ALLOW_PUBLIC_KEY_RETRIEVAL)) { - // Return AWS RDS Auth provider return Optional.of(new AwsRdsDatabaseAuthenticationProvider()); + } else if (jdbcURL.contains(AzureDatabaseAuthenticationProvider.AZURE)) { + return Optional.of(new AzureDatabaseAuthenticationProvider()); } // Return empty diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/JdbiUtils.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/JdbiUtils.java new file mode 100644 index 000000000000..ce471b7868fc --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/jdbi/JdbiUtils.java @@ -0,0 +1,55 @@ +package org.openmetadata.service.util.jdbi; + +import com.codahale.metrics.NoopMetricRegistry; +import io.dropwizard.db.DataSourceFactory; +import io.dropwizard.jdbi3.JdbiFactory; +import io.dropwizard.setup.Environment; +import org.jdbi.v3.core.Jdbi; +import org.jdbi.v3.core.statement.SqlStatements; +import org.jdbi.v3.sqlobject.SqlObjectPlugin; +import org.jdbi.v3.sqlobject.SqlObjects; +import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator; + +public class JdbiUtils { + + public static Jdbi createAndSetupJDBI(Environment environment, DataSourceFactory dbFactory) { + DatabaseAuthenticationProviderFactory.get(dbFactory.getUrl()) + .ifPresent( + databaseAuthenticationProvider -> { + String token = + databaseAuthenticationProvider.authenticate( + dbFactory.getUrl(), dbFactory.getUser(), dbFactory.getPassword()); + dbFactory.setPassword(token); + }); + + Jdbi jdbiInstance = new JdbiFactory().build(environment, dbFactory, "database"); + jdbiInstance.setSqlLogger(new OMSqlLogger()); + // Set the Database type for choosing correct queries from annotations + jdbiInstance + .getConfig(SqlObjects.class) + .setSqlLocator(new ConnectionAwareAnnotationSqlLocator(dbFactory.getDriverClass())); + jdbiInstance.getConfig(SqlStatements.class).setUnusedBindingAllowed(true); + + return jdbiInstance; + } + + public static Jdbi createAndSetupJDBI(DataSourceFactory dbFactory) { + DatabaseAuthenticationProviderFactory.get(dbFactory.getUrl()) + .ifPresent( + databaseAuthenticationProvider -> { + String token = + databaseAuthenticationProvider.authenticate( + dbFactory.getUrl(), dbFactory.getUser(), dbFactory.getPassword()); + dbFactory.setPassword(token); + }); + + Jdbi jdbiInstance = Jdbi.create(dbFactory.build(new NoopMetricRegistry(), "open-metadata-ops")); + jdbiInstance.installPlugin(new SqlObjectPlugin()); + jdbiInstance + .getConfig(SqlObjects.class) + .setSqlLocator(new ConnectionAwareAnnotationSqlLocator(dbFactory.getDriverClass())); + jdbiInstance.getConfig(SqlStatements.class).setUnusedBindingAllowed(true); + + return jdbiInstance; + } +} diff --git a/pom.xml b/pom.xml index 54d7c4207c71..5b61176bc5ec 100644 --- a/pom.xml +++ b/pom.xml @@ -551,6 +551,17 @@ ${jsonpath.version} + + com.microsoft.azure + msal4j + 1.17.2 + + + com.azure + azure-identity + 1.14.0 + +