Merge branch 'main' into debuggability-alert-notifications

openmetadata-docs/content/v1.5.x/deployment/ingestion/openmetadata.md

@@ -119,9 +119,8 @@ openmetadata:
 ## Custom Airflow Installation
 
 {% note %}
-- Note that the `openmetadata-ingestion` only supports Python versions 3.7, 3.8 and 3.9.
-- The supported Airflow versions are 2.3, 2.4 and 2.5. From release 1.1.1 onwards, OpenMetadata will also support Airflow 2.6.
-{% /note %}
+- Note that the `openmetadata-ingestion` only supports Python versions 3.7, 3.8, 3.9 and 3.10.
+- The supported Airflow versions are 2.3, 2.4, 2.5, 2.6, and 2.7. Starting from release 1.5, OpenMetadata will support compatibility with Airflow versions up to 2.9.
 
 You will need to follow three steps:
 1. Install the `openmetadata-ingestion` package with the connector plugins that you need.

openmetadata-docs/content/v1.6.x-SNAPSHOT/deployment/ingestion/openmetadata.md

@@ -119,8 +119,8 @@ openmetadata:
 ## Custom Airflow Installation
 
 {% note %}
-- Note that the `openmetadata-ingestion` only supports Python versions 3.7, 3.8 and 3.9.
-- The supported Airflow versions are 2.3, 2.4 and 2.5. From release 1.1.1 onwards, OpenMetadata will also support Airflow 2.6.
+- Note that the `openmetadata-ingestion` only supports Python versions 3.7, 3.8, 3.9 and 3.10.
+- - The supported Airflow versions are 2.3, 2.4, 2.5, 2.6, and 2.7. Starting from release 1.5, OpenMetadata will support compatibility with Airflow versions up to 2.9.
 {% /note %}
 
 You will need to follow three steps:

openmetadata-service/src/main/java/org/openmetadata/service/Entity.java

@@ -122,6 +122,7 @@ public final class Entity {
   public static final String FIELD_STYLE = "style";
 
   public static final String FIELD_LIFE_CYCLE = "lifeCycle";
+  public static final String FIELD_CERTIFICATION = "certification";
 
   public static final String FIELD_DISABLED = "disabled";
 

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java

@@ -65,6 +65,7 @@
 import org.openmetadata.schema.auth.PersonalAccessToken;
 import org.openmetadata.schema.auth.RefreshToken;
 import org.openmetadata.schema.auth.TokenType;
+import org.openmetadata.schema.configuration.AssetCertificationSettings;
 import org.openmetadata.schema.dataInsight.DataInsightChart;
 import org.openmetadata.schema.dataInsight.custom.DataInsightCustomChart;
 import org.openmetadata.schema.dataInsight.kpi.Kpi;
@@ -4964,6 +4965,8 @@ public static Settings getSettings(SettingsType configType, String json) {
                 .readValue(json, String.class);
             case PROFILER_CONFIGURATION -> JsonUtils.readValue(json, ProfilerConfiguration.class);
             case SEARCH_SETTINGS -> JsonUtils.readValue(json, SearchSettings.class);
+            case ASSET_CERTIFICATION_SETTINGS -> JsonUtils.readValue(
+                json, AssetCertificationSettings.class);
             default -> throw new IllegalArgumentException("Invalid Settings Type " + configType);
           };
       settings.setConfigValue(value);

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseRepository.java

@@ -13,6 +13,7 @@
 
 package org.openmetadata.service.jdbi3;
 
+import static org.openmetadata.csv.CsvUtil.addExtension;
 import static org.openmetadata.csv.CsvUtil.addField;
 import static org.openmetadata.csv.CsvUtil.addGlossaryTerms;
 import static org.openmetadata.csv.CsvUtil.addOwners;
@@ -68,6 +69,7 @@ public DatabaseRepository() {
         "",
         "");
     supportsSearch = true;
+    fieldFetchers.put("name", this::fetchAndSetService);
   }
 
   @Override
@@ -125,7 +127,8 @@ public String exportToCsv(String name, String user) throws IOException {
         (DatabaseSchemaRepository) Entity.getEntityRepository(DATABASE_SCHEMA);
     List<DatabaseSchema> schemas =
         repository.listAllForCSV(
-            repository.getFields("owners,tags,domain"), database.getFullyQualifiedName());
+            repository.getFields("owners,tags,domain,extension"), database.getFullyQualifiedName());
+
     schemas.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName));
     return new DatabaseCsv(database, user).exportCsv(schemas);
   }
@@ -224,6 +227,17 @@ public Database deleteDatabaseProfilerConfig(UUID databaseId) {
     return database;
   }
 
+  private void fetchAndSetService(List<Database> entities, Fields fields) {
+    if (entities == null || entities.isEmpty() || (!fields.contains("name"))) {
+      return;
+    }
+
+    EntityReference service = getContainer(entities.get(0).getId());
+    for (Database database : entities) {
+      database.setService(service);
+    }
+  }
+
   public class DatabaseUpdater extends EntityUpdater {
     public DatabaseUpdater(Database original, Database updated, Operation operation) {
       super(original, updated, operation);
@@ -282,7 +296,8 @@ protected void createEntity(CSVPrinter printer, List<CSVRecord> csvRecords) thro
           .withTags(tagLabels)
           .withRetentionPeriod(csvRecord.get(7))
           .withSourceUrl(csvRecord.get(8))
-          .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN));
+          .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN))
+          .withExtension(getExtension(printer, csvRecord, 10));
       if (processRecord) {
         createEntity(printer, csvRecord, schema);
       }
@@ -306,6 +321,7 @@ protected void addRecord(CsvFile csvFile, DatabaseSchema entity) {
               ? ""
               : entity.getDomain().getFullyQualifiedName();
       addField(recordList, domain);
+      addExtension(recordList, entity.getExtension());
       addRecord(csvFile, recordList);
     }
   }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseSchemaRepository.java

@@ -14,6 +14,7 @@
 package org.openmetadata.service.jdbi3;
 
 import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty;
+import static org.openmetadata.csv.CsvUtil.addExtension;
 import static org.openmetadata.csv.CsvUtil.addField;
 import static org.openmetadata.csv.CsvUtil.addGlossaryTerms;
 import static org.openmetadata.csv.CsvUtil.addOwners;
@@ -196,9 +197,11 @@ public void entityRelationshipReindex(DatabaseSchema original, DatabaseSchema up
   public String exportToCsv(String name, String user) throws IOException {
     DatabaseSchema schema = getByName(null, name, Fields.EMPTY_FIELDS); // Validate database schema
     TableRepository repository = (TableRepository) Entity.getEntityRepository(TABLE);
+
     List<Table> tables =
         repository.listAllForCSV(
-            repository.getFields("owners,tags,domain"), schema.getFullyQualifiedName());
+            repository.getFields("owners,tags,domain,extension"), schema.getFullyQualifiedName());
+
     tables.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName));
     return new DatabaseSchemaCsv(schema, user).exportCsv(tables);
   }
@@ -315,7 +318,8 @@ protected void createEntity(CSVPrinter printer, List<CSVRecord> csvRecords) thro
           .withRetentionPeriod(csvRecord.get(7))
           .withSourceUrl(csvRecord.get(8))
           .withColumns(nullOrEmpty(table.getColumns()) ? new ArrayList<>() : table.getColumns())
-          .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN));
+          .withDomain(getEntityReference(printer, csvRecord, 9, Entity.DOMAIN))
+          .withExtension(getExtension(printer, csvRecord, 10));
 
       if (processRecord) {
         createEntity(printer, csvRecord, table);
@@ -340,6 +344,7 @@ protected void addRecord(CsvFile csvFile, Table entity) {
               ? ""
               : entity.getDomain().getFullyQualifiedName();
       addField(recordList, domain);
+      addExtension(recordList, entity.getExtension());
       addRecord(csvFile, recordList);
     }
   }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DatabaseServiceRepository.java

@@ -13,6 +13,7 @@
 
 package org.openmetadata.service.jdbi3;
 
+import static org.openmetadata.csv.CsvUtil.addExtension;
 import static org.openmetadata.csv.CsvUtil.addField;
 import static org.openmetadata.csv.CsvUtil.addGlossaryTerms;
 import static org.openmetadata.csv.CsvUtil.addOwners;
@@ -68,7 +69,9 @@ public String exportToCsv(String name, String user) throws IOException {
     DatabaseRepository repository = (DatabaseRepository) Entity.getEntityRepository(DATABASE);
     List<Database> databases =
         repository.listAllForCSV(
-            repository.getFields("owners,tags,domain"), databaseService.getFullyQualifiedName());
+            repository.getFields("name,owners,tags,domain,extension"),
+            databaseService.getFullyQualifiedName());
+
     databases.sort(Comparator.comparing(EntityInterface::getFullyQualifiedName));
     return new DatabaseServiceCsv(databaseService, user).exportCsv(databases);
   }
@@ -122,7 +125,8 @@ protected void createEntity(CSVPrinter printer, List<CSVRecord> csvRecords) thro
           .withDescription(csvRecord.get(2))
           .withOwners(getOwners(printer, csvRecord, 3))
           .withTags(tagLabels)
-          .withDomain(getEntityReference(printer, csvRecord, 7, Entity.DOMAIN));
+          .withDomain(getEntityReference(printer, csvRecord, 7, Entity.DOMAIN))
+          .withExtension(getExtension(printer, csvRecord, 8));
 
       if (processRecord) {
         createEntity(printer, csvRecord, database);
@@ -145,6 +149,7 @@ protected void addRecord(CsvFile csvFile, Database entity) {
               ? ""
               : entity.getDomain().getFullyQualifiedName();
       addField(recordList, domain);
+      addExtension(recordList, entity.getExtension());
       addRecord(csvFile, recordList);
     }
   }

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java

@@ -29,6 +29,7 @@
 import static org.openmetadata.service.Entity.ADMIN_USER_NAME;
 import static org.openmetadata.service.Entity.DATA_PRODUCT;
 import static org.openmetadata.service.Entity.DOMAIN;
+import static org.openmetadata.service.Entity.FIELD_CERTIFICATION;
 import static org.openmetadata.service.Entity.FIELD_CHILDREN;
 import static org.openmetadata.service.Entity.FIELD_DATA_PRODUCTS;
 import static org.openmetadata.service.Entity.FIELD_DELETED;
@@ -78,8 +79,11 @@
 import com.networknt.schema.ValidationMessage;
 import java.io.IOException;
 import java.net.URI;
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
+import java.time.Period;
+import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 import java.time.temporal.TemporalAccessor;
@@ -121,12 +125,14 @@
 import org.openmetadata.schema.api.VoteRequest.VoteType;
 import org.openmetadata.schema.api.feed.ResolveTask;
 import org.openmetadata.schema.api.teams.CreateTeam;
+import org.openmetadata.schema.configuration.AssetCertificationSettings;
 import org.openmetadata.schema.entity.data.Table;
 import org.openmetadata.schema.entity.feed.Suggestion;
 import org.openmetadata.schema.entity.teams.Team;
 import org.openmetadata.schema.entity.teams.User;
 import org.openmetadata.schema.system.EntityError;
 import org.openmetadata.schema.type.ApiStatus;
+import org.openmetadata.schema.type.AssetCertification;
 import org.openmetadata.schema.type.ChangeDescription;
 import org.openmetadata.schema.type.ChangeEvent;
 import org.openmetadata.schema.type.Column;
@@ -235,6 +241,7 @@ public record EntityHistoryWithOffset(EntityHistory entityHistory, int nextOffse
   @Getter protected final boolean supportsOwners;
   @Getter protected final boolean supportsStyle;
   @Getter protected final boolean supportsLifeCycle;
+  @Getter protected final boolean supportsCertification;
   protected final boolean supportsFollower;
   protected final boolean supportsExtension;
   protected final boolean supportsVotes;
@@ -328,6 +335,11 @@ protected EntityRepository(
       this.patchFields.addField(allowedFields, FIELD_LIFE_CYCLE);
       this.putFields.addField(allowedFields, FIELD_LIFE_CYCLE);
     }
+    this.supportsCertification = allowedFields.contains(FIELD_CERTIFICATION);
+    if (supportsCertification) {
+      this.patchFields.addField(allowedFields, FIELD_CERTIFICATION);
+      this.putFields.addField(allowedFields, FIELD_CERTIFICATION);
+    }
 
     Map<String, Pair<Boolean, BiConsumer<List<T>, Fields>>> fieldSupportMap = new HashMap<>();
 
@@ -2627,6 +2639,7 @@ private void updateInternal() {
         updateReviewers();
         updateStyle();
         updateLifeCycle();
+        updateCertification();
         entitySpecificUpdate();
       }
     }
@@ -2929,6 +2942,53 @@ private void updateLifeCycle() {
       recordChange(FIELD_LIFE_CYCLE, origLifeCycle, updatedLifeCycle, true);
     }
 
+    private void updateCertification() {
+      if (!supportsCertification) {
+        return;
+      }
+      AssetCertification origCertification = original.getCertification();
+      AssetCertification updatedCertification = updated.getCertification();
+
+      if (origCertification == updatedCertification || updatedCertification == null) return;
+
+      SystemRepository systemRepository = Entity.getSystemRepository();
+      AssetCertificationSettings assetCertificationSettings =
+          systemRepository.getAssetCertificationSettings();
+
+      String certificationLabel = updatedCertification.getTagLabel().getTagFQN();
+
+      validateCertification(certificationLabel, assetCertificationSettings);
+
+      long certificationDate = System.currentTimeMillis();
+      updatedCertification.setAppliedDate(certificationDate);
+
+      LocalDateTime nowDateTime =
+          LocalDateTime.ofInstant(Instant.ofEpochMilli(certificationDate), ZoneOffset.UTC);
+      Period datePeriod = Period.parse(assetCertificationSettings.getValidityPeriod());
+      LocalDateTime targetDateTime = nowDateTime.plus(datePeriod);
+      updatedCertification.setExpiryDate(targetDateTime.toInstant(ZoneOffset.UTC).toEpochMilli());
+
+      recordChange(FIELD_CERTIFICATION, origCertification, updatedCertification, true);
+    }
+
+    private void validateCertification(
+        String certificationLabel, AssetCertificationSettings assetCertificationSettings) {
+      if (Optional.ofNullable(assetCertificationSettings).isEmpty()) {
+        throw new IllegalArgumentException(
+            "Certification is not configured. Please configure the Classification used for Certification in the Settings.");
+      } else {
+        String allowedClassification = assetCertificationSettings.getAllowedClassification();
+        String[] fqnParts = FullyQualifiedName.split(certificationLabel);
+        String parentFqn = FullyQualifiedName.getParentFQN(fqnParts);
+        if (!allowedClassification.equals(parentFqn)) {
+          throw new IllegalArgumentException(
+              String.format(
+                  "Invalid Classification: %s is not valid for Certification.",
+                  certificationLabel));
+        }
+      }
+    }
+
     public final boolean updateVersion(Double oldVersion) {
       Double newVersion = oldVersion;
       if (majorVersionChange) {

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java

@@ -9,13 +9,15 @@
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Optional;
 import javax.json.JsonPatch;
 import javax.json.JsonValue;
 import javax.ws.rs.core.Response;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.jdbi.v3.sqlobject.transaction.Transaction;
 import org.openmetadata.api.configuration.UiThemePreference;
+import org.openmetadata.schema.configuration.AssetCertificationSettings;
 import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.entity.services.ingestionPipelines.PipelineServiceClientResponse;
 import org.openmetadata.schema.security.client.OpenMetadataJWTClientConfig;
@@ -110,6 +112,15 @@ public Settings getConfigWithKey(String key) {
     return null;
   }
 
+  public AssetCertificationSettings getAssetCertificationSettings() {
+    Optional<Settings> oAssetCertificationSettings =
+        Optional.ofNullable(getConfigWithKey(SettingsType.ASSET_CERTIFICATION_SETTINGS.value()));
+
+    return oAssetCertificationSettings
+        .map(settings -> (AssetCertificationSettings) settings.getConfigValue())
+        .orElse(null);
+  }
+
   public Settings getEmailConfigInternal() {
     try {
       Settings setting = dao.getConfigWithKey(SettingsType.EMAIL_CONFIGURATION.value());

openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java

@@ -13,6 +13,7 @@
 
 package org.openmetadata.service.resources.settings;
 
+import static org.openmetadata.schema.settings.SettingsType.ASSET_CERTIFICATION_SETTINGS;
 import static org.openmetadata.schema.settings.SettingsType.CUSTOM_UI_THEME_PREFERENCE;
 import static org.openmetadata.schema.settings.SettingsType.EMAIL_CONFIGURATION;
 import static org.openmetadata.schema.settings.SettingsType.LOGIN_CONFIGURATION;
@@ -30,6 +31,7 @@
 import org.openmetadata.api.configuration.UiThemePreference;
 import org.openmetadata.schema.api.configuration.LoginConfiguration;
 import org.openmetadata.schema.api.searcg.SearchSettings;
+import org.openmetadata.schema.configuration.AssetCertificationSettings;
 import org.openmetadata.schema.email.SmtpSettings;
 import org.openmetadata.schema.settings.Settings;
 import org.openmetadata.schema.settings.SettingsType;
@@ -124,6 +126,20 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app
               .withConfigValue(new SearchSettings().withEnableAccessControl(false));
       systemRepository.createNewSetting(setting);
     }
+
+    // Initialise Certification Settings
+    Settings certificationSettings =
+        systemRepository.getConfigWithKey(ASSET_CERTIFICATION_SETTINGS.toString());
+    if (certificationSettings == null) {
+      Settings setting =
+          new Settings()
+              .withConfigType(ASSET_CERTIFICATION_SETTINGS)
+              .withConfigValue(
+                  new AssetCertificationSettings()
+                      .withAllowedClassification("Certification")
+                      .withValidityPeriod("P30D"));
+      systemRepository.createNewSetting(setting);
+    }
   }
 
   public static <T> T getSetting(SettingsType settingName, Class<T> clazz) {

openmetadata-service/src/main/resources/json/data/database/databaseCsvDocumentation.json

@@ -84,6 +84,23 @@
       "examples": [
         "Marketing", "Sales"
       ]
+    },
+    {
+      "name": "extension",
+      "required": false,
+      "description": "Custom property values added to the glossary term. Each field value (property and its value) is separated by `;` and internal values can be separated by `|`. For `entityReferenceList` type property, pass `type1:fqn1|type2:fqn2`. For single `entityReference` type property, pass `type:fqn`. Similarly, for `enumMultiSelect`, pass values separated by `|`, and for `enumSingleSelect`, pass a single value along with the property name. For `timeInterval` property type, pass the `startTime:endTime` to the property name. If the field value itself contains delimiter values like `,` and `;` or newline they need to be quoted, and the quotation needs to be further escaped. In general, if passing multiple field values separated by `;`, the extension column value needs to be quoted.",
+      "examples": [
+        "`customAttribute1:value1;customAttribute2:value2`",
+        "`\"dateCp:18-09-2024;dateTimeCp:18-09-2024 01:09:34;durationCp:PT5H30M10S;emailCp:admin@open-metadata.org\"`",
+        "`entRefListCp:searchIndex:elasticsearch_sample.table_search_index|databaseSchema:Glue.default.information_schema|databaseSchema:sample_data.ecommerce_db.shopify|database:Glue.default|`",
+        "`\"entRefCp:user:\"\"aaron.singh2\"\"\"`",
+        "`\"enumMultiSelectCp:val3|val2|val1|val4|val5;enumSingleSelectCp:singleVal1\"`",
+        "`\"timeCp:10:08:45;timeIntervalCp:1726142300000:17261420000;timeStampCp:1726142400000\"`",
+        "`\"integerCp:7777;numberCp:123456\"`",
+        "`\"\"\"queryCp:select col,row from table where id ='30';\"\";stringcp:sample string content\"`",
+        "`markdownCp:# Sample Markdown Text`",
+        "\"\"\"tableCp:row_1_col1_Value,row_1_col2_Value,row_1_col3_Value\"\"\""
+      ]
     }
   ]
 }