open-metadata · dasomel · Nov 18, 2024 · Nov 19, 2024 · Dec 10, 2024 · Dec 10, 2024
@@ -5,13 +5,14 @@ run-name: OpenMetadata release cherry-pick PR #${{ github.event.pull_request.num
 
 # yamllint disable-line rule:truthy
 on:
-  pull_request:
+  pull_request_target:
     types: [closed]
     branches:
       - main
 permissions:
   contents: write
   pull-requests: write
+  issues: write
 env:
   CURRENT_RELEASE_ENDPOINT: ${{ vars.CURRENT_RELEASE_ENDPOINT }}  # Endpoint that returns the current release version in json format
 jobs:

@@ -31,9 +31,9 @@ jobs:
   playwright-ci-mysql:
     runs-on: ubuntu-latest
     strategy:
-    fail-fast: false
-    matrix:
-      shardIndex: [1, 2]
-      shardTotal: [2]
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2]
+        shardTotal: [2]
     steps:
       - run: 'echo "Step is not required"'
@@ -36,7 +36,7 @@ concurrency:
 
 jobs:
   playwright-ci-mysql:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: test
     strategy:
       fail-fast: false

@@ -31,9 +31,9 @@ jobs:
   playwright-ci-postgresql:
     runs-on: ubuntu-latest
     strategy:
-    fail-fast: false
-    matrix:
-      shardIndex: [1, 2]
-      shardTotal: [2]
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2]
+        shardTotal: [2]
     steps:
       - run: 'echo "Step is not required"'
@@ -36,7 +36,7 @@ concurrency:
 
 jobs:
   playwright-ci-postgresql:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     environment: test
     strategy:
       fail-fast: false

diff --git a/bootstrap/sql/migrations/native/1.5.15/mysql/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.15/mysql/postDataMigrationSQLScript.sql
diff --git a/bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.5.15/mysql/schemaChanges.sql
@@ -0,0 +1,5 @@
+-- Make domain policy and role non-system
+UPDATE policy_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessPolicy';
+UPDATE policy_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessPolicy';
+UPDATE role_entity SET json = JSON_SET(json, '$.provider', 'user') where name = 'DomainOnlyAccessRole';
+UPDATE role_entity SET json = JSON_SET(json, '$.allowDelete', true) where name = 'DomainOnlyAccessRole';
diff --git a/bootstrap/sql/migrations/native/1.5.15/postgres/postDataMigrationSQLScript.sql b/bootstrap/sql/migrations/native/1.5.15/postgres/postDataMigrationSQLScript.sql
diff --git a/bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.5.15/postgres/schemaChanges.sql
@@ -0,0 +1,5 @@
+-- Make domain policy and role non-system
+UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessPolicy';
+UPDATE policy_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessPolicy';
+UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{provider}', '"user"', true) where name = 'DomainOnlyAccessRole';
+UPDATE role_entity SET json = JSONB_SET(json::jsonb, '{allowDelete}', 'true', true) WHERE name = 'DomainOnlyAccessRole';
diff --git a/bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql
@@ -0,0 +1,44 @@
+-- add timestamp index for test case result reindex performance
+ALTER TABLE data_quality_data_time_series ADD INDEX `idx_timestamp_desc` (timestamp DESC);
+
+CREATE TABLE background_jobs (
+  id BIGINT UNSIGNED AUTO_INCREMENT PRIMARY KEY,
+  jobType VARCHAR(256) NOT NULL,
+  methodName VARCHAR(256) NOT NULL,
+  jobArgs JSON NOT NULL,
+  status VARCHAR(50) NOT NULL DEFAULT 'PENDING',
+  createdBy VARCHAR(256) NOT NULL,
+  createdAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000),
+  updatedAt BIGINT UNSIGNED NOT NULL DEFAULT (UNIX_TIMESTAMP(NOW(3)) * 1000)
+);
+
+CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt);
+CREATE INDEX idx_createdBy ON background_jobs (createdBy);
+CREATE INDEX idx_status ON background_jobs (status);
+CREATE INDEX idx_jobType ON background_jobs (jobType);
+CREATE INDEX idx_updatedAt ON background_jobs (updatedAt);
+
+-- rename executable -> basic for test suites
+UPDATE test_suite
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.executable'),
+    '$.basic',
+    JSON_EXTRACT(json, '$.executable')
+)
+WHERE JSON_EXTRACT(json, '$.executable') IS NOT NULL;
+
+-- rename executableEntityReference -> basicEntityReference for test suites
+UPDATE test_suite
+SET json = JSON_INSERT(
+    JSON_REMOVE(json, '$.executableEntityReference'),
+    '$.basicEntityReference',
+    JSON_EXTRACT(json, '$.executableEntityReference')
+)
+WHERE JSON_EXTRACT(json, '$.executableEntityReference') IS NOT NULL;
+
+-- clean up the testSuites
+UPDATE test_case SET json = json_remove(json, '$.testSuites');
+
+-- clean up the testSuites in the version history too
+UPDATE entity_extension SET json = json_remove(json, '$.testSuites') WHERE jsonSchema = 'testCase';
+
diff --git a/bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql
@@ -0,0 +1,46 @@
+-- add timestamp index for test case result reindex performance
+CREATE INDEX idx_timestamp_desc ON data_quality_data_time_series (timestamp DESC);
+
+CREATE TABLE background_jobs (
+  id BIGSERIAL PRIMARY KEY,
+  jobType VARCHAR(256) NOT NULL,
+  methodName VARCHAR(256) NOT NULL,
+  jobArgs JSONB NOT NULL,
+  status VARCHAR(50) NOT NULL DEFAULT 'PENDING',
+  createdBy VARCHAR(256) NOT NULL,
+  createdAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT,
+  updatedAt BIGINT NOT NULL DEFAULT (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT
+);
+
+CREATE INDEX idx_status_createdAt ON background_jobs (status, createdAt);
+CREATE INDEX idx_createdBy ON background_jobs (createdBy);
+CREATE INDEX idx_status ON background_jobs (status);
+CREATE INDEX idx_jobType ON background_jobs (jobType);
+CREATE INDEX idx_updatedAt ON background_jobs (updatedAt);
+
+-- rename executable -> basic for test suites
+UPDATE test_suite
+SET json = jsonb_set(
+  json::jsonb #- '{executable}',
+  '{basic}',
+  (json #> '{executable}')::jsonb,
+  true
+)
+WHERE json #>> '{executable}' IS NOT NULL;
+
+-- rename executableEntityReference -> basicEntityReference for test suites
+UPDATE test_suite
+SET json = jsonb_set(
+  json::jsonb #- '{executableEntityReference}',
+  '{basicEntityReference}',
+  (json #> '{executableEntityReference}')::jsonb,
+  true
+)
+WHERE json #>> '{executableEntityReference}' IS NOT NULL;
+
+-- clean up the testSuites
+UPDATE test_case SET json = json::jsonb #- '{testSuites}';
+
+-- clean up the testSuites in the version history too
+UPDATE entity_extension SET json = json::jsonb #- '{testSuites}' WHERE jsonSchema = 'testCase';
+
diff --git a/bootstrap/sql/migrations/native/1.6.3/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.3/mysql/schemaChanges.sql
@@ -0,0 +1,6 @@
+-- Add constraint to apps_data_store
+ALTER TABLE apps_data_store ADD CONSTRAINT entity_relationship_pky PRIMARY KEY (identifier, type);
+UPDATE user_entity SET json = JSON_SET(json, '$.isBot', false) WHERE JSON_EXTRACT(json, '$.isBot') IS NULL;
+ALTER TABLE user_entity  ADD COLUMN isBot BOOLEAN GENERATED ALWAYS AS (json -> '$.isBot') NOT NULL;
+CREATE INDEX idx_isBot ON user_entity (isBot);
+DELETE from apps_data_store;
diff --git a/bootstrap/sql/migrations/native/1.6.3/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.3/postgres/schemaChanges.sql
@@ -0,0 +1,6 @@
+-- Add constraint to apps_data_store
+ALTER TABLE apps_data_store ADD CONSTRAINT entity_relationship_pky PRIMARY KEY (identifier, type);
+UPDATE user_entity SET json = jsonb_set(json::jsonb, '{isBot}', 'false'::jsonb, true) WHERE NOT (json ?? 'isBot');
+ALTER TABLE user_entity ADD COLUMN isBot BOOLEAN GENERATED ALWAYS AS ((json ->> 'deleted')::boolean) STORED NOT NULL;
+CREATE INDEX idx_isBot ON user_entity (isBot);
+DELETE from apps_data_store;
diff --git a/common/pom.xml b/common/pom.xml
@@ -18,7 +18,7 @@
   <parent>
     <artifactId>platform</artifactId>
     <groupId>org.open-metadata</groupId>
-    <version>1.6.0-SNAPSHOT</version>
+    <version>1.6.3</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <properties>

diff --git a/common/src/main/java/org/openmetadata/annotations/DeprecatedAnnotator.java b/common/src/main/java/org/openmetadata/annotations/DeprecatedAnnotator.java
@@ -0,0 +1,76 @@
+package org.openmetadata.annotations;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.sun.codemodel.JAnnotationUse;
+import com.sun.codemodel.JClass;
+import com.sun.codemodel.JDefinedClass;
+import com.sun.codemodel.JFieldVar;
+import com.sun.codemodel.JMethod;
+import java.lang.reflect.Field;
+import java.util.TreeMap;
+import org.jsonschema2pojo.AbstractAnnotator;
+
+/** Add {@link Deprecated} annotation to generated Java classes */
+public class DeprecatedAnnotator extends AbstractAnnotator {
+
+  /** Add {@link Deprecated} annotation to property fields */
+  @Override
+  public void propertyField(
+      JFieldVar field, JDefinedClass clazz, String propertyName, JsonNode propertyNode) {
+    super.propertyField(field, clazz, propertyName, propertyNode);
+    if (propertyNode.get("deprecated") != null && propertyNode.get("deprecated").asBoolean()) {
+      field.annotate(Deprecated.class);
+    }
+  }
+
+  /** Add {@link Deprecated} annotation to getter methods */
+  @Override
+  public void propertyGetter(JMethod getter, JDefinedClass clazz, String propertyName) {
+    super.propertyGetter(getter, clazz, propertyName);
+    addDeprecatedAnnotationIfApplies(getter, propertyName);
+  }
+
+  /** Add {@link Deprecated} annotation to setter methods */
+  @Override
+  public void propertySetter(JMethod setter, JDefinedClass clazz, String propertyName) {
+    super.propertySetter(setter, clazz, propertyName);
+    addDeprecatedAnnotationIfApplies(setter, propertyName);
+  }
+
+  /**
+   * Use reflection methods to access the {@link JDefinedClass} of the {@link JMethod} object. If
+   * the {@link JMethod} is pointing to a field annotated with {@link Deprecated} then annotates
+   * the {@link JMethod} object with {@link Deprecated}
+   */
+  private void addDeprecatedAnnotationIfApplies(JMethod jMethod, String propertyName) {
+    try {
+      Field outerClassField = JMethod.class.getDeclaredField("outer");
+      outerClassField.setAccessible(true);
+      JDefinedClass outerClass = (JDefinedClass) outerClassField.get(jMethod);
+
+      TreeMap<String, JFieldVar> insensitiveFieldsMap =
+          new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+      insensitiveFieldsMap.putAll(outerClass.fields());
+
+      if (insensitiveFieldsMap.containsKey(propertyName)
+          && insensitiveFieldsMap.get(propertyName).annotations().stream()
+              .anyMatch(
+                  annotation ->
+                      Deprecated.class.getName().equals(getAnnotationClassName(annotation)))) {
+        jMethod.annotate(Deprecated.class);
+      }
+    } catch (NoSuchFieldException | IllegalAccessException e) {
+      throw new RuntimeException(e);
+    }
+  }
+
+  private String getAnnotationClassName(JAnnotationUse annotation) {
+    try {
+      Field clazzField = JAnnotationUse.class.getDeclaredField("clazz");
+      clazzField.setAccessible(true);
+      return ((JClass) clazzField.get(annotation)).fullName();
+    } catch (NoSuchFieldException | IllegalAccessException e) {
+      throw new RuntimeException(e);
+    }
+  }
+}
diff --git a/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java b/common/src/main/java/org/openmetadata/annotations/OpenMetadataAnnotator.java
@@ -19,6 +19,10 @@ public class OpenMetadataAnnotator extends CompositeAnnotator {
 
   public OpenMetadataAnnotator() {
     // we can add multiple annotators
-    super(new ExposedAnnotator(), new MaskedAnnotator(), new PasswordAnnotator());
+    super(
+        new ExposedAnnotator(),
+        new MaskedAnnotator(),
+        new PasswordAnnotator(),
+        new DeprecatedAnnotator());
   }
 }
diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml
@@ -180,6 +180,7 @@ authenticationConfiguration:
   # This will only be valid when provider type specified is customOidc
   providerName: ${CUSTOM_OIDC_AUTHENTICATION_PROVIDER_NAME:-""}
   publicKeyUrls: ${AUTHENTICATION_PUBLIC_KEYS:-[http://localhost:8585/api/v1/system/config/jwks]}
+  tokenValidationAlgorithm: ${AUTHENTICATION_TOKEN_VALIDATION_ALGORITHM:-"RS256"}
   authority: ${AUTHENTICATION_AUTHORITY:-https://accounts.google.com}
   clientId: ${AUTHENTICATION_CLIENT_ID:-""}
   callbackUrl: ${AUTHENTICATION_CALLBACK_URL:-""}
@@ -295,6 +296,7 @@ eventMonitoringConfiguration:
   batchSize: ${EVENT_MONITOR_BATCH_SIZE:-10}
   pathPattern: ${EVENT_MONITOR_PATH_PATTERN:-["/api/v1/tables/*", "/api/v1/health-check"]}
   latency: ${EVENT_MONITOR_LATENCY:-[0.99, 0.90]} # For value p99=0.99, p90=0.90, p50=0.50 etc.
+  servicesHealthCheckInterval: ${EVENT_MONITOR_SERVICES_HEALTH_CHECK_INTERVAL:-300}
   # it will use the default auth provider for AWS services if parameters are not set
   # parameters:
   #   region: ${OM_MONITOR_REGION:-""}

@@ -18,7 +18,7 @@ volumes:
 services:  
   ingestion:
     container_name: openmetadata_ingestion
-    image: docker.getcollate.io/openmetadata/ingestion:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/ingestion:1.6.3
     environment:
       AIRFLOW__API__AUTH_BACKENDS: "airflow.api.auth.backend.basic_auth,airflow.api.auth.backend.session"
       AIRFLOW__CORE__EXECUTOR: LocalExecutor

@@ -14,7 +14,7 @@ services:
   execute-migrate-all:
     container_name: execute_migrate_all
     command: "./bootstrap/openmetadata-ops.sh migrate"
-    image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/server:1.6.3
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}
@@ -227,7 +227,7 @@ services:
   openmetadata-server:
     container_name: openmetadata_server
     restart: always
-    image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/server:1.6.3
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}

@@ -11,7 +11,7 @@
 
 # Build stage
 FROM alpine:3 AS build
-ARG RI_VERSION="1.6.0-SNAPSHOT"
+ARG RI_VERSION="1.6.3"
 ENV RELEASE_URL="https://github.com/open-metadata/OpenMetadata/releases/download/${RI_VERSION}-release/openmetadata-${RI_VERSION}.tar.gz"
 
 RUN mkdir -p /opt/openmetadata && \
@@ -21,7 +21,7 @@ RUN mkdir -p /opt/openmetadata && \
 
 # Final stage
 FROM alpine:3
-ARG RI_VERSION="1.6.0-SNAPSHOT"
+ARG RI_VERSION="1.6.3"
 ARG BUILD_DATE
 ARG COMMIT_ID
 LABEL maintainer="OpenMetadata"

@@ -18,7 +18,7 @@ volumes:
 services:
   postgresql:
     container_name: openmetadata_postgresql
-    image: docker.getcollate.io/openmetadata/postgresql:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/postgresql:1.6.3
     restart: always
     command: "--work_mem=10MB"
     environment:
@@ -61,7 +61,7 @@ services:
 
   execute-migrate-all:
     container_name: execute_migrate_all
-    image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/server:1.6.3
     command: "./bootstrap/openmetadata-ops.sh migrate"
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
@@ -275,7 +275,7 @@ services:
   openmetadata-server:
     container_name: openmetadata_server
     restart: always
-    image: docker.getcollate.io/openmetadata/server:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/server:1.6.3
     environment:
       OPENMETADATA_CLUSTER_NAME: ${OPENMETADATA_CLUSTER_NAME:-openmetadata}
       SERVER_PORT: ${SERVER_PORT:-8585}
@@ -483,7 +483,7 @@ services:
 
   ingestion:
     container_name: openmetadata_ingestion
-    image: docker.getcollate.io/openmetadata/ingestion:1.5.0-SNAPSHOT
+    image: docker.getcollate.io/openmetadata/ingestion:1.6.3
     depends_on:
       elasticsearch:
         condition: service_started