From 594db310f3c22a0612d6c071fbc9b4b4119f4c62 Mon Sep 17 00:00:00 2001 From: prasar-ashutosh Date: Tue, 4 Jul 2023 21:30:18 +0800 Subject: [PATCH] Persistence Component - Big query support (#1904) --- .../DeriveMainDatasetSchemaFromStaging.java | 7 +- .../IngestModeOptimizationColumnHandler.java | 7 +- ...plicator.java => DatasetDeduplicator.java} | 41 +- .../logicalplan/LogicalPlanFactory.java | 13 +- .../logicalplan/datasets/DataType.java | 6 +- .../datasets/PartitionKeyAbstract.java | 35 + .../datasets/SchemaDefinitionAbstract.java | 2 + .../logicalplan/values/FunctionName.java | 9 +- .../planner/NontemporalDeltaPlanner.java | 17 +- .../components/planner/Planner.java | 2 +- .../planner/UnitemporalDeltaPlanner.java | 4 +- .../components/util/LogicalPlanUtils.java | 20 +- .../schemaevolution/SchemaEvolution.java | 68 +- .../persistence/components/sink/Sink.java | 6 + .../relational/ansi/AnsiSqlSink.java | 11 + .../sql/visitors/DerivedDatasetVisitor.java | 18 +- .../ansi/sql/visitors/FieldVisitor.java | 9 +- .../ansi/sql/visitors/SQLCreateVisitor.java | 2 +- .../ansi/sql/visitors/SelectionVisitor.java | 36 +- .../nontemporal/NontemporalDeltaTest.java | 14 +- .../UnitemporalDeltaBatchIdBasedTest.java | 34 +- .../components/util/LogicalPlanUtilsTest.java | 6 +- .../pom.xml | 128 ++ .../relational/bigquery/BigQuerySink.java | 231 ++++ .../bigquery/executor/BigQueryConnection.java | 38 + .../bigquery/executor/BigQueryExecutor.java | 146 ++ .../bigquery/executor/BigQueryHelper.java | 418 ++++++ .../executor/BigQueryTransactionManager.java | 179 +++ .../bigquery/sql/BigQueryDataTypeMapping.java | 121 ++ ...QueryDataTypeToLogicalDataTypeMapping.java | 71 + .../bigquery/sql/visitor/AlterVisitor.java | 43 + .../sql/visitor/BatchEndTimestampVisitor.java | 44 + .../visitor/BatchStartTimestampVisitor.java | 43 + .../sql/visitor/ClusterKeyVisitor.java | 40 + .../bigquery/sql/visitor/DeleteVisitor.java | 52 + .../bigquery/sql/visitor/FieldVisitor.java | 26 + .../sql/visitor/PartitionKeyVisitor.java | 40 + .../sql/visitor/SQLCreateVisitor.java | 60 + .../sql/visitor/SchemaDefinitionVisitor.java | 84 ++ .../bigquery/sql/visitor/TruncateVisitor.java | 47 + .../columns/PKColumnConstraint.java | 26 + .../bigquery/sqldom/schema/Bool.java | 26 + .../bigquery/sqldom/schema/Bytes.java | 32 + .../bigquery/sqldom/schema/Float64.java | 27 + .../bigquery/sqldom/schema/Json.java | 26 + .../bigquery/sqldom/schema/String.java | 32 + .../schemaops/statements/AlterTable.java | 132 ++ .../schemaops/statements/CreateTable.java | 173 +++ .../persistence/components/BaseTestUtils.java | 179 +++ .../e2e/AppendOnlyExecutorTest.java | 86 ++ .../e2e/AppendOnlyGeneratorTest.java | 69 + .../components/e2e/BigQueryEndToEndTest.java | 422 ++++++ .../e2e/BitempDeltaExecutorTest.java | 166 +++ .../e2e/BitempDeltaGeneratorTest.java | 125 ++ .../e2e/NonTemporalDeltaExecutorTest.java | 76 ++ .../e2e/NonTemporalDeltaGeneratorTest.java | 67 + .../e2e/NonTemporalSnapshotExecutorTest.java | 84 ++ .../e2e/NonTemporalSnapshotGeneratorTest.java | 67 + .../components/e2e/SchemaEvolutionTest.java | 628 +++++++++ .../e2e/UnitempDeltaExecutorTest.java | 88 ++ .../e2e/UnitempDeltaGeneratorTest.java | 72 + .../e2e/UnitempSnapshotExecutorTest.java | 93 ++ .../e2e/UnitempSnapshotGeneratorTest.java | 73 + .../components/ingestmode/AppendOnlyTest.java | 258 ++++ .../ingestmode/BigQueryTestArtifacts.java | 343 +++++ ...eltaSourceSpecifiesFromAndThroughTest.java | 293 ++++ ...itemporalDeltaSourceSpecifiesFromTest.java | 1177 +++++++++++++++++ .../components/ingestmode/IngestModeTest.java | 222 ++++ .../ingestmode/NontemporalDeltaTest.java | 350 +++++ .../ingestmode/NontemporalSnapshotTest.java | 175 +++ .../UnitemporalDeltaBatchIdBasedTest.java | 480 +++++++ ...temporalDeltaBatchIdDateTimeBasedTest.java | 462 +++++++ .../UnitemporalDeltaDateTimeBasedTest.java | 232 ++++ .../UnitemporalSnapshotBatchIdBasedTest.java | 177 +++ ...poralSnapshotBatchIdDateTimeBasedTest.java | 233 ++++ .../UnitemporalSnapshotDateTimeBasedTest.java | 193 +++ .../logicalplan/operations/AlterTest.java | 97 ++ .../operations/CreateTableTest.java | 256 ++++ .../bigquery/sql/DataTypeMappingTest.java | 109 ++ .../transformer/PlaceholderTest.java | 77 ++ .../resources/expected/append/data_pass1.csv | 3 + .../resources/expected/append/data_pass2.csv | 5 + .../expected/bitemp_delta/data_pass1.csv | 1 + .../expected/bitemp_delta/data_pass2.csv | 3 + .../expected/bitemp_delta/data_pass3.csv | 5 + .../expected/bitemp_delta/data_pass4.csv | 7 + .../expected/bitemp_delta/data_pass5.csv | 8 + .../expected/bitemp_delta/data_pass6.csv | 9 + .../expected/nontemporal_delta/data_pass1.csv | 3 + .../expected/nontemporal_delta/data_pass2.csv | 4 + .../nontemporal_snapshot/data_pass1.csv | 3 + .../nontemporal_snapshot/data_pass2.csv | 3 + .../expected/unitemp_delta/data_pass1.csv | 3 + .../expected/unitemp_delta/data_pass2.csv | 5 + .../expected/unitemp_snapshot/data_pass1.csv | 3 + .../expected/unitemp_snapshot/data_pass2.csv | 5 + .../input/bitemp_delta/data_pass1.csv | 1 + .../input/bitemp_delta/data_pass2.csv | 1 + .../input/bitemp_delta/data_pass3.csv | 1 + .../input/bitemp_delta/data_pass4.csv | 1 + .../input/bitemp_delta/data_pass5.csv | 1 + .../input/bitemp_delta/data_pass6.csv | 0 .../src/test/resources/input/data_pass1.csv | 3 + .../src/test/resources/input/data_pass2.csv | 3 + .../components/relational/RelationalSink.java | 65 +- .../relational/api/RelationalConnection.java | 19 + .../api/RelationalIngestorAbstract.java | 10 +- .../executor/RelationalExecutionHelper.java | 50 + .../executor/RelationalExecutor.java | 29 +- .../jdbc/JdbcConnectionAbstract.java | 34 + .../relational/jdbc/JdbcHelper.java | 29 +- .../jdbc/JdbcTransactionManager.java | 14 +- ...dbcPropertiesToLogicalDataTypeMapping.java | 2 +- .../relational/sqldom/common/Clause.java | 4 +- .../sqldom/common/FunctionName.java | 8 + .../table/PartitionKeyConstraint.java | 39 + .../table/PrimaryKeyTableConstraint.java | 18 +- .../relational/sqldom/schemaops/Column.java | 6 + .../schemaops/statements/InsertStatement.java | 14 +- .../schemaops/values/WindowFunction.java | 9 +- .../sqldom/schemaops/InsertTest.java | 2 +- .../components/relational/h2/H2Sink.java | 26 + ...dbcPropertiesToLogicalDataTypeMapping.java | 2 +- .../h2/sql/visitor/FieldVisitor.java | 26 + .../persistence/components/BaseTest.java | 7 +- .../persistence/components/TestUtils.java | 8 +- .../relational/jdbc/JdbcHelperTest.java | 3 +- .../relational/memsql/MemSqlSink.java | 22 +- .../memsql/sql/visitor/FieldVisitor.java | 26 + .../memsql/sql/visitor/SQLCreateVisitor.java | 2 +- .../ingestmode/NontemporalDeltaTest.java | 16 +- .../UnitemporalDeltaBatchIdBasedTest.java | 38 +- .../relational/snowflake/SnowflakeSink.java | 24 + ...dbcPropertiesToLogicalDataTypeMapping.java | 2 +- .../snowflake/sql/visitor/FieldVisitor.java | 26 + .../sql/visitor/SQLCreateVisitor.java | 2 +- .../ingestmode/NontemporalDeltaMergeTest.java | 5 +- .../pom.xml | 1 + .../extension/PersistenceTestRunner.java | 3 +- 139 files changed, 10492 insertions(+), 261 deletions(-) rename legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/{DatasetFilterAndDeduplicator.java => DatasetDeduplicator.java} (73%) create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/PartitionKeyAbstract.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/pom.xml create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/BigQuerySink.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryConnection.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryExecutor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryHelper.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryTransactionManager.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/BigQueryDataTypeMapping.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/BigQueryDataTypeToLogicalDataTypeMapping.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/AlterVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchEndTimestampVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchStartTimestampVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/ClusterKeyVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/DeleteVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/FieldVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/PartitionKeyVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/SQLCreateVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/SchemaDefinitionVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/TruncateVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/constraints/columns/PKColumnConstraint.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Bool.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Bytes.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Float64.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Json.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/String.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/AlterTable.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/CreateTable.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/BaseTestUtils.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/AppendOnlyExecutorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/AppendOnlyGeneratorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BigQueryEndToEndTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BitempDeltaExecutorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BitempDeltaGeneratorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalDeltaExecutorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalDeltaGeneratorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalSnapshotExecutorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalSnapshotGeneratorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/SchemaEvolutionTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempDeltaExecutorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempDeltaGeneratorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempSnapshotExecutorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempSnapshotGeneratorTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/AppendOnlyTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BigQueryTestArtifacts.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BitemporalDeltaSourceSpecifiesFromAndThroughTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BitemporalDeltaSourceSpecifiesFromTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalSnapshotTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdBasedTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdDateTimeBasedTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaDateTimeBasedTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdDateTimeBasedTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotDateTimeBasedTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/AlterTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/CreateTableTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/DataTypeMappingTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/transformer/PlaceholderTest.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/append/data_pass1.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/append/data_pass2.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass1.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass2.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass3.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass4.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass5.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass6.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_delta/data_pass1.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_delta/data_pass2.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_snapshot/data_pass1.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_snapshot/data_pass2.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_delta/data_pass1.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_delta/data_pass2.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_snapshot/data_pass1.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_snapshot/data_pass2.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass1.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass2.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass3.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass4.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass5.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass6.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/data_pass1.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/data_pass2.csv create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/RelationalConnection.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/executor/RelationalExecutionHelper.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcConnectionAbstract.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/constraints/table/PartitionKeyConstraint.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/visitor/FieldVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/sql/visitor/FieldVisitor.java create mode 100644 legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/FieldVisitor.java diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/DeriveMainDatasetSchemaFromStaging.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/DeriveMainDatasetSchemaFromStaging.java index 0187769c761..293daac0e41 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/DeriveMainDatasetSchemaFromStaging.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/DeriveMainDatasetSchemaFromStaging.java @@ -33,7 +33,12 @@ import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.derivation.SourceSpecifiesFromAndThruDateTimeAbstract; import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.derivation.SourceSpecifiesFromDateTimeAbstract; import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.derivation.ValidityDerivationVisitor; -import org.finos.legend.engine.persistence.components.logicalplan.datasets.*; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; import java.util.ArrayList; import java.util.List; diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeOptimizationColumnHandler.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeOptimizationColumnHandler.java index 4da6689ef59..151822fb31e 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeOptimizationColumnHandler.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeOptimizationColumnHandler.java @@ -22,7 +22,8 @@ import java.util.List; import java.util.stream.Collectors; -import static org.finos.legend.engine.persistence.components.util.LogicalPlanUtils.*; +import static org.finos.legend.engine.persistence.components.util.LogicalPlanUtils.SUPPORTED_DATA_TYPES_FOR_OPTIMIZATION_COLUMNS; +import static org.finos.legend.engine.persistence.components.util.LogicalPlanUtils.findCommonPrimaryFieldsBetweenMainAndStaging; public class IngestModeOptimizationColumnHandler implements IngestModeVisitor { @@ -92,9 +93,9 @@ private List deriveOptimizationFilters(UnitemporalDeltaAbstr List primaryKeys = findCommonPrimaryFieldsBetweenMainAndStaging(datasets.mainDataset(), datasets.stagingDataset()); List comparablePrimaryKeys = primaryKeys.stream().filter(field -> SUPPORTED_DATA_TYPES_FOR_OPTIMIZATION_COLUMNS.contains(field.type().dataType())).collect(Collectors.toList()); optimizationFilters = new ArrayList<>(); - for (Field field: comparablePrimaryKeys) + for (Field field : comparablePrimaryKeys) { - OptimizationFilter filter = OptimizationFilter.of(field.name(),field.name().toUpperCase() + "_LOWER", field.name().toUpperCase() + "_UPPER"); + OptimizationFilter filter = OptimizationFilter.of(field.name(), field.name().toUpperCase() + "_LOWER", field.name().toUpperCase() + "_UPPER"); optimizationFilters.add(filter); } } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetFilterAndDeduplicator.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetDeduplicator.java similarity index 73% rename from legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetFilterAndDeduplicator.java rename to legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetDeduplicator.java index 7bdf8b9892b..275515bb448 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetFilterAndDeduplicator.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetDeduplicator.java @@ -18,39 +18,37 @@ import org.finos.legend.engine.persistence.components.logicalplan.conditions.Equals; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Selection; -import org.finos.legend.engine.persistence.components.logicalplan.values.*; -import org.finos.legend.engine.persistence.components.util.LogicalPlanUtils; +import org.finos.legend.engine.persistence.components.logicalplan.values.FieldValue; +import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionImpl; +import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionName; +import org.finos.legend.engine.persistence.components.logicalplan.values.ObjectValue; +import org.finos.legend.engine.persistence.components.logicalplan.values.Order; +import org.finos.legend.engine.persistence.components.logicalplan.values.OrderedField; +import org.finos.legend.engine.persistence.components.logicalplan.values.Value; +import org.finos.legend.engine.persistence.components.logicalplan.values.WindowFunction; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.stream.Collectors; -public class DatasetFilterAndDeduplicator implements VersioningStrategyVisitor +public class DatasetDeduplicator implements VersioningStrategyVisitor { Dataset stagingDataset; List primaryKeys; - Optional stagingDatasetFilter; private static final String ROW_NUMBER = "legend_persistence_row_num"; - public DatasetFilterAndDeduplicator(Dataset stagingDataset, List primaryKeys) + public DatasetDeduplicator(Dataset stagingDataset, List primaryKeys) { this.stagingDataset = stagingDataset; this.primaryKeys = primaryKeys; - this.stagingDatasetFilter = LogicalPlanUtils.getDatasetFilterCondition(stagingDataset); } @Override public Dataset visitNoVersioningStrategy(NoVersioningStrategyAbstract noVersioningStrategy) { - Dataset enrichedStagingDataset = this.stagingDataset; - if (this.stagingDatasetFilter.isPresent()) - { - enrichedStagingDataset = filterDataset(); - } - return enrichedStagingDataset; + return this.stagingDataset; } @Override @@ -78,7 +76,6 @@ public Dataset visitMaxVersionStrategy(MaxVersionStrategyAbstract maxVersionStra Selection selectionWithRowNumber = Selection.builder() .source(stagingDataset) .addAllFields(allColumnsWithRowNumber) - .condition(stagingDatasetFilter) .alias(stagingDataset.datasetReference().alias()) .build(); @@ -91,22 +88,6 @@ public Dataset visitMaxVersionStrategy(MaxVersionStrategyAbstract maxVersionStra .alias(stagingDataset.datasetReference().alias()) .build(); } - else if (this.stagingDatasetFilter.isPresent()) - { - enrichedStagingDataset = filterDataset(); - } return enrichedStagingDataset; } - - private Dataset filterDataset() - { - List allColumns = new ArrayList<>(stagingDataset.schemaReference().fieldValues()); - Selection selection = Selection.builder() - .source(this.stagingDataset) - .addAllFields(allColumns) - .condition(this.stagingDatasetFilter.get()) - .alias(stagingDataset.datasetReference().alias()) - .build(); - return selection; - } } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/LogicalPlanFactory.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/LogicalPlanFactory.java index 525ebc84683..33ae2fcad34 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/LogicalPlanFactory.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/LogicalPlanFactory.java @@ -105,17 +105,10 @@ public static LogicalPlan getLogicalPlanForNextBatchId(Datasets datasets) public static LogicalPlan getLogicalPlanForMinAndMaxForField(Dataset dataset, String fieldName) { FieldValue field = FieldValue.builder().datasetRef(dataset.datasetReference()).fieldName(fieldName).build(); - Selection.Builder selectionBuilder = Selection.builder() + Selection selection = Selection.builder() .addFields(FunctionImpl.builder().functionName(FunctionName.MIN).addValue(field).alias(MIN_OF_FIELD).build()) .addFields(FunctionImpl.builder().functionName(FunctionName.MAX).addValue(field).alias(MAX_OF_FIELD).build()) - .source(dataset); - - Optional filterCondition = LogicalPlanUtils.getDatasetFilterCondition(dataset); - if (filterCondition.isPresent()) - { - selectionBuilder = selectionBuilder.condition(filterCondition); - } - - return LogicalPlan.builder().addOps(selectionBuilder.build()).build(); + .source(dataset).build(); + return LogicalPlan.builder().addOps(selection).build(); } } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/DataType.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/DataType.java index d099590d62f..d2c2c4d9d53 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/DataType.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/DataType.java @@ -56,6 +56,8 @@ public enum DataType LONGNVARCHAR, UNDEFINED, INT64, + FLOAT64, + BYTES, STRING, BOOL, LONGTEXT, @@ -67,13 +69,13 @@ public enum DataType public static boolean isStringDatatype(DataType type) { - List stringDatatype = new ArrayList(Arrays.asList(CHAR, CHARACTER, VARCHAR, LONGVARCHAR, NCHAR, NVARCHAR, LONGNVARCHAR, LONGTEXT, TEXT, JSON, STRING)); + List stringDatatype = new ArrayList<>(Arrays.asList(CHAR, CHARACTER, VARCHAR, LONGVARCHAR, NCHAR, NVARCHAR, LONGNVARCHAR, LONGTEXT, TEXT, JSON, STRING)); return stringDatatype.contains(type); } public static Set getComparableDataTypes() { - return new HashSet<>(Arrays.asList(INT, INTEGER, BIGINT, TINYINT, SMALLINT, INT64, REAL, DECIMAL, FLOAT, DOUBLE, NUMBER, NUMERIC, + return new HashSet<>(Arrays.asList(INT, INTEGER, BIGINT, TINYINT, SMALLINT, INT64, FLOAT64, REAL, DECIMAL, FLOAT, DOUBLE, NUMBER, NUMERIC, TIME, TIMESTAMP, TIMESTAMP_NTZ, TIMESTAMP_TZ, TIMESTAMP_LTZ, DATETIME, TIMESTAMPTZ, DATE)); } } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/PartitionKeyAbstract.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/PartitionKeyAbstract.java new file mode 100644 index 00000000000..54b23a6c841 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/PartitionKeyAbstract.java @@ -0,0 +1,35 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.logicalplan.datasets; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanNode; +import org.finos.legend.engine.persistence.components.logicalplan.values.Value; +import org.immutables.value.Value.Immutable; +import org.immutables.value.Value.Parameter; +import org.immutables.value.Value.Style; + +@Immutable +@Style( + typeAbstract = "*Abstract", + typeImmutable = "*", + jdkOnly = true, + optionalAcceptNullable = true, + strictBuilder = true +) +public interface PartitionKeyAbstract extends LogicalPlanNode +{ + @Parameter(order = 0) + Value key(); +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/SchemaDefinitionAbstract.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/SchemaDefinitionAbstract.java index e71dddd4105..549b6f4306d 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/SchemaDefinitionAbstract.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/datasets/SchemaDefinitionAbstract.java @@ -36,6 +36,8 @@ public interface SchemaDefinitionAbstract extends Schema List clusterKeys(); + List partitionKeys(); + Optional columnStoreSpecification(); Optional shardSpecification(); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/FunctionName.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/FunctionName.java index 5fa4a653fda..110c39a7de6 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/FunctionName.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/logicalplan/values/FunctionName.java @@ -28,5 +28,12 @@ public enum FunctionName UPPER, ROW_NUMBER, SUBSTRING, - PARSE_JSON; + PARSE_JSON, + DATE, + DATE_TRUNC, + DATETIME_TRUNC, + TIMESTAMP_TRUNC, + RANGE_BUCKET, + GENERATE_ARRAY, + PARSE_DATETIME; } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/NontemporalDeltaPlanner.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/NontemporalDeltaPlanner.java index 6c305f116c9..76d3ce59bf6 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/NontemporalDeltaPlanner.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/NontemporalDeltaPlanner.java @@ -20,7 +20,7 @@ import org.finos.legend.engine.persistence.components.common.StatisticName; import org.finos.legend.engine.persistence.components.ingestmode.NontemporalDelta; import org.finos.legend.engine.persistence.components.ingestmode.audit.AuditingVisitors; -import org.finos.legend.engine.persistence.components.ingestmode.deduplication.DatasetFilterAndDeduplicator; +import org.finos.legend.engine.persistence.components.ingestmode.deduplication.DatasetDeduplicator; import org.finos.legend.engine.persistence.components.ingestmode.deduplication.VersioningConditionVisitor; import org.finos.legend.engine.persistence.components.ingestmode.merge.MergeStrategyVisitors; import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; @@ -93,7 +93,7 @@ class NontemporalDeltaPlanner extends Planner // Perform Deduplication & Filtering of Staging Dataset this.enrichedStagingDataset = ingestMode().versioningStrategy() - .accept(new DatasetFilterAndDeduplicator(stagingDataset(), primaryKeys)); + .accept(new DatasetDeduplicator(stagingDataset(), primaryKeys)); } @Override @@ -194,6 +194,12 @@ private Merge getMergeOperation() versioningCondition = this.versioningCondition; } + if (ingestMode().auditing().accept(AUDIT_ENABLED)) + { + String auditField = ingestMode().auditing().accept(AuditingVisitors.EXTRACT_AUDIT_FIELD).orElseThrow(IllegalStateException::new); + keyValuePairs.add(Pair.of(FieldValue.builder().datasetRef(mainDataset().datasetReference()).fieldName(auditField).build(), batchStartTimestamp)); + } + Merge merge = Merge.builder() .dataset(mainDataset()) .usingDataset(stagingDataset) @@ -203,13 +209,6 @@ private Merge getMergeOperation() .matchedCondition(versioningCondition) .build(); - if (ingestMode().auditing().accept(AUDIT_ENABLED)) - { - String auditField = ingestMode().auditing().accept(AuditingVisitors.EXTRACT_AUDIT_FIELD).orElseThrow(IllegalStateException::new); - keyValuePairs.add(Pair.of(FieldValue.builder().datasetRef(mainDataset().datasetReference()).fieldName(auditField).build(), batchStartTimestamp)); - merge = merge.withUnmatchedKeyValuePairs(keyValuePairs); - } - return merge; } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/Planner.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/Planner.java index 93178497076..9edfc56265e 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/Planner.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/Planner.java @@ -210,7 +210,7 @@ protected void addPreRunStatsForRowsDeleted(Map preR protected void addPostRunStatsForIncomingRecords(Map postRunStatisticsResult) { - Optional filterCondition = LogicalPlanUtils.getDatasetFilterCondition(stagingDataset()); + Optional filterCondition = Optional.empty(); if (dataSplitExecutionSupported()) { Optional dataSplitInRangeCondition = getDataSplitInRangeConditionForStatistics(); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/UnitemporalDeltaPlanner.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/UnitemporalDeltaPlanner.java index ecf9a0cbab9..8c7c1282234 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/UnitemporalDeltaPlanner.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/planner/UnitemporalDeltaPlanner.java @@ -18,7 +18,7 @@ import org.finos.legend.engine.persistence.components.common.Resources; import org.finos.legend.engine.persistence.components.common.StatisticName; import org.finos.legend.engine.persistence.components.ingestmode.UnitemporalDelta; -import org.finos.legend.engine.persistence.components.ingestmode.deduplication.DatasetFilterAndDeduplicator; +import org.finos.legend.engine.persistence.components.ingestmode.deduplication.DatasetDeduplicator; import org.finos.legend.engine.persistence.components.ingestmode.deduplication.VersioningConditionVisitor; import org.finos.legend.engine.persistence.components.ingestmode.merge.MergeStrategyVisitors; import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; @@ -84,7 +84,7 @@ class UnitemporalDeltaPlanner extends UnitemporalPlanner this.dataSplitInRangeCondition = ingestMode.dataSplitField().map(field -> LogicalPlanUtils.getDataSplitInRangeCondition(stagingDataset(), field)); // Perform Deduplication & Filtering of Staging Dataset this.enrichedStagingDataset = ingestMode().versioningStrategy() - .accept(new DatasetFilterAndDeduplicator(stagingDataset(), primaryKeys)); + .accept(new DatasetDeduplicator(stagingDataset(), primaryKeys)); this.versioningCondition = ingestMode().versioningStrategy() .accept(new VersioningConditionVisitor(mainDataset(), stagingDataset(), false, ingestMode().digestField())); this.inverseVersioningCondition = ingestMode.versioningStrategy() diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtils.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtils.java index 0b275ec151c..8828f5261ec 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtils.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtils.java @@ -244,21 +244,15 @@ public static void replaceField(List fieldsList, String oldFieldName, Str }); } - public static Optional getDatasetFilterCondition(Dataset dataSet) + public static Condition getDatasetFilterCondition(DerivedDataset derivedDataset) { - Optional filter = Optional.empty(); - if (dataSet instanceof DerivedDataset) + List datasetFilters = derivedDataset.datasetFilters(); + List conditions = new ArrayList<>(); + for (DatasetFilter datasetFilter: datasetFilters) { - DerivedDataset derivedDataset = (DerivedDataset) dataSet; - List datasetFilters = derivedDataset.datasetFilters(); - List conditions = new ArrayList<>(); - for (DatasetFilter datasetFilter: datasetFilters) - { - conditions.add(datasetFilter.mapFilterToCondition(dataSet.datasetReference())); - } - filter = Optional.of(And.of(conditions)); + conditions.add(datasetFilter.mapFilterToCondition(derivedDataset.datasetReference())); } - return filter; + return And.of(conditions); } public static List getDatasetFilters(Dataset dataSet) @@ -320,7 +314,7 @@ public static Selection getRecordCount(Dataset dataset, String alias) public static Selection getRecordCount(Dataset dataset, String alias, Optional condition) { return Selection.builder() - .source(dataset.datasetReference()) + .source(dataset) .addFields(FunctionImpl.builder().functionName(FunctionName.COUNT).alias(alias).addValue(All.INSTANCE).build()) .condition(condition) .build(); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/src/main/java/org/finos/legend/engine/persistence/components/schemaevolution/SchemaEvolution.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/src/main/java/org/finos/legend/engine/persistence/components/schemaevolution/SchemaEvolution.java index 1e5305f274d..edbb9e0a779 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/src/main/java/org/finos/legend/engine/persistence/components/schemaevolution/SchemaEvolution.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/src/main/java/org/finos/legend/engine/persistence/components/schemaevolution/SchemaEvolution.java @@ -46,8 +46,8 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Optional; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -120,9 +120,9 @@ private void validatePrimaryKeys(Dataset mainDataset, Dataset stagingDataset) //Validate all columns (allowing exceptions) in staging dataset must have a matching column in main dataset private List stagingToMainTableColumnMatch(Dataset mainDataset, - Dataset stagingDataset, - Set fieldsToIgnore, - Set modifiedFields) + Dataset stagingDataset, + Set fieldsToIgnore, + Set modifiedFields) { List operations = new ArrayList<>(); List mainFields = mainDataset.schema().fields(); @@ -159,7 +159,7 @@ private List stagingToMainTableColumnMatch(Dataset mainDataset, if (sink.capabilities().contains(Capability.IMPLICIT_DATA_TYPE_CONVERSION) && sink.supportsImplicitMapping(matchedMainField.type().dataType(), stagingFieldType.dataType())) { - Field newField = evolveFieldLength(stagingField, matchedMainField); + Field newField = sink.evolveFieldLength(stagingField, matchedMainField); evolveDataType(newField, matchedMainField, mainDataset, operations, modifiedFields); } // If the datatype is a non-breaking change, we alter the datatype. @@ -170,7 +170,7 @@ else if (sink.capabilities().contains(Capability.EXPLICIT_DATA_TYPE_CONVERSION) if (schemaEvolutionCapabilitySet.contains(SchemaEvolutionCapability.DATA_TYPE_CONVERSION)) { //Modify the column in main table - Field newField = evolveFieldLength(matchedMainField, stagingField); + Field newField = sink.evolveFieldLength(matchedMainField, stagingField); evolveDataType(newField, matchedMainField, mainDataset, operations, modifiedFields); } else @@ -188,7 +188,7 @@ else if (sink.capabilities().contains(Capability.EXPLICIT_DATA_TYPE_CONVERSION) //If data types are same, we check if length requires any evolution else { - Field newField = evolveFieldLength(stagingField, matchedMainField); + Field newField = sink.evolveFieldLength(stagingField, matchedMainField); evolveDataType(newField, matchedMainField, mainDataset, operations, modifiedFields); } } @@ -197,7 +197,7 @@ else if (sink.capabilities().contains(Capability.EXPLICIT_DATA_TYPE_CONVERSION) { if (!matchedMainField.nullable() && stagingField.nullable()) { - Field newField = createNewField(matchedMainField, stagingField, matchedMainField.type().length(), matchedMainField.type().scale()); + Field newField = sink.createNewField(matchedMainField, stagingField, matchedMainField.type().length(), matchedMainField.type().scale()); evolveDataType(newField, matchedMainField, mainDataset, operations, modifiedFields); } } @@ -224,7 +224,7 @@ private void evolveDataType(Field newField, Field mainDataField, Dataset mainDat if (!Objects.equals(mainDataField.type().scale(), newField.type().scale())) { if (!sink.capabilities().contains(Capability.DATA_TYPE_SCALE_CHANGE) - || (!schemaEvolutionCapabilitySet.contains(SchemaEvolutionCapability.DATA_TYPE_SIZE_CHANGE))) + || (!schemaEvolutionCapabilitySet.contains(SchemaEvolutionCapability.DATA_TYPE_SIZE_CHANGE))) { throw new IncompatibleSchemaChangeException(String.format("Data type scale changes couldn't be performed on column \"%s\" since user capability does not allow it", newField.name())); } @@ -281,56 +281,6 @@ private List mainToStagingTableColumnMatch(Dataset mainDataset, Datas return operations; } - //new field = field to replace main column (datatype) - //old field = reference field to compare sizing/nullability requirements - private Field evolveFieldLength(Field oldField, Field newField) - { - Optional length = newField.type().length(); - Optional scale = newField.type().scale(); - - //If the oldField and newField have a length associated, pick the greater length - if (oldField.type().length().isPresent() && newField.type().length().isPresent()) - { - length = newField.type().length().get() >= oldField.type().length().get() - ? newField.type().length() - : oldField.type().length(); - } - //Allow length evolution from unspecified length only when data types are same. This is to avoid evolution like SMALLINT(6) -> INT(6) or INT -> DOUBLE(6) and allow for DATETIME -> DATETIME(6) - else if (oldField.type().dataType().equals(newField.type().dataType()) - && oldField.type().length().isPresent() && !newField.type().length().isPresent()) - { - length = oldField.type().length(); - } - - //If the oldField and newField have a scale associated, pick the greater scale - if (oldField.type().scale().isPresent() && newField.type().scale().isPresent()) - { - scale = newField.type().scale().get() >= oldField.type().scale().get() - ? newField.type().scale() - : oldField.type().scale(); - } - //Allow scale evolution from unspecified scale only when data types are same. This is to avoid evolution like SMALLINT(6) -> INT(6) or INT -> DOUBLE(6) and allow for DATETIME -> DATETIME(6) - else if (oldField.type().dataType().equals(newField.type().dataType()) - && oldField.type().scale().isPresent() && !newField.type().scale().isPresent()) - { - scale = oldField.type().scale(); - } - return createNewField(newField, oldField, length, scale); - } - - - private Field createNewField(Field newField, Field oldField, Optional length, Optional scale) - { - FieldType modifiedFieldType = FieldType.of(newField.type().dataType(), length, scale); - boolean nullability = newField.nullable() || oldField.nullable(); - - //todo : how to handle default value, identity, uniqueness ? - return Field.builder().name(newField.name()).primaryKey(newField.primaryKey()) - .fieldAlias(newField.fieldAlias()).nullable(nullability) - .identity(newField.identity()).unique(newField.unique()) - .defaultValue(newField.defaultValue()).type(modifiedFieldType).build(); - } - private SchemaDefinition evolveSchemaDefinition(SchemaDefinition schema, Set modifiedFields) { final Set modifiedFieldNames = modifiedFields.stream().map(Field::name).collect(Collectors.toSet()); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/src/main/java/org/finos/legend/engine/persistence/components/sink/Sink.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/src/main/java/org/finos/legend/engine/persistence/components/sink/Sink.java index 1d369f8a27a..38f2862d886 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/src/main/java/org/finos/legend/engine/persistence/components/sink/Sink.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-physical-plan/src/main/java/org/finos/legend/engine/persistence/components/sink/Sink.java @@ -16,9 +16,11 @@ import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanNode; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; import org.finos.legend.engine.persistence.components.util.Capability; +import java.util.Optional; import java.util.Set; public interface Sink @@ -32,4 +34,8 @@ public interface Sink String quoteIdentifier(); LogicalPlanVisitor visitorForClass(Class clazz); + + Field evolveFieldLength(Field evolveFrom, Field evolveTo); + + Field createNewField(Field evolveTo, Field evolveFrom, Optional length, Optional scale); } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/AnsiSqlSink.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/AnsiSqlSink.java index 17b5943d8be..7211c711cb2 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/AnsiSqlSink.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/AnsiSqlSink.java @@ -14,6 +14,7 @@ package org.finos.legend.engine.persistence.components.relational.ansi; +import org.finos.legend.engine.persistence.components.executor.Executor; import org.finos.legend.engine.persistence.components.logicalplan.conditions.And; import org.finos.legend.engine.persistence.components.logicalplan.conditions.Equals; import org.finos.legend.engine.persistence.components.logicalplan.conditions.Exists; @@ -77,6 +78,7 @@ import org.finos.legend.engine.persistence.components.optimizer.Optimizer; import org.finos.legend.engine.persistence.components.relational.CaseConversion; import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.LowerCaseOptimizer; import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.UpperCaseOptimizer; import org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.AllQuantifierVisitor; @@ -135,6 +137,9 @@ import org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.TableConstraintVisitor; import org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.WindowFunctionVisitor; import org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.ParseJsonFunctionVisitor; +import org.finos.legend.engine.persistence.components.relational.api.RelationalConnection; +import org.finos.legend.engine.persistence.components.relational.sql.TabularData; +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; import org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils; import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; import org.finos.legend.engine.persistence.components.util.Capability; @@ -286,6 +291,12 @@ public Optional optimizerForCaseConversion(CaseConversion caseConvers } } + @Override + public Executor getRelationalExecutor(RelationalConnection connection) + { + throw new UnsupportedOperationException("No executor supported for AnsiSql Sink"); + } + // utility methods private static Map, LogicalPlanVisitor> rightBiasedUnion(Map, LogicalPlanVisitor> map1, Map, LogicalPlanVisitor> map2) diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/DerivedDatasetVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/DerivedDatasetVisitor.java index bd0a92667c2..b598c89851e 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/DerivedDatasetVisitor.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/DerivedDatasetVisitor.java @@ -14,16 +14,32 @@ package org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors; +import org.finos.legend.engine.persistence.components.logicalplan.conditions.Condition; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DerivedDataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Selection; +import org.finos.legend.engine.persistence.components.logicalplan.values.Value; import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; import org.finos.legend.engine.persistence.components.transformer.VisitorContext; +import org.finos.legend.engine.persistence.components.util.LogicalPlanUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; public class DerivedDatasetVisitor implements LogicalPlanVisitor { @Override public VisitorResult visit(PhysicalPlanNode prev, DerivedDataset current, VisitorContext context) { - return new DatasetReferenceVisitor().visit(prev, current.datasetReference(), context); + Condition filterCondition = LogicalPlanUtils.getDatasetFilterCondition(current); + List allColumns = new ArrayList<>(current.schemaReference().fieldValues()); + Selection selection = Selection.builder() + .source(current.datasetReference()) + .addAllFields(allColumns) + .condition(filterCondition) + .alias(current.datasetReference().alias()) + .build(); + return new SelectionVisitor().visit(prev, selection, context); } } \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/FieldVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/FieldVisitor.java index d705270a197..d50c1e5f4ed 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/FieldVisitor.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/FieldVisitor.java @@ -18,6 +18,7 @@ import org.finos.legend.engine.persistence.components.optimizer.Optimizer; import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; import org.finos.legend.engine.persistence.components.relational.ansi.sql.AnsiDatatypeMapping; +import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.ColumnConstraint; import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.NotNullColumnConstraint; import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.PKColumnConstraint; @@ -36,7 +37,7 @@ public class FieldVisitor implements LogicalPlanVisitor @Override public VisitorResult visit(PhysicalPlanNode prev, Field current, VisitorContext context) { - DataType dataType = new AnsiDatatypeMapping().getDataType(current.type()); + DataType dataType = getDataTypeMapping().getDataType(current.type()); List columnConstraints = new ArrayList<>(); if (!current.nullable()) { @@ -61,4 +62,10 @@ public VisitorResult visit(PhysicalPlanNode prev, Field current, VisitorContext return new VisitorResult(null); } + + public DataTypeMapping getDataTypeMapping() + { + return new AnsiDatatypeMapping(); + } + } \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/SQLCreateVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/SQLCreateVisitor.java index c48952acc39..91e0b444517 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/SQLCreateVisitor.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/SQLCreateVisitor.java @@ -35,7 +35,7 @@ public VisitorResult visit(PhysicalPlanNode prev, Create current, VisitorContext prev.push(createTable); List logicalPlanNodes = new ArrayList<>(); - logicalPlanNodes.add(current.dataset()); + logicalPlanNodes.add(current.dataset().datasetReference()); logicalPlanNodes.add(current.dataset().schema()); if (current.ifNotExists()) diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/SelectionVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/SelectionVisitor.java index f6b5fc2c2d4..07806398ad8 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/SelectionVisitor.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/main/java/org/finos/legend/engine/persistence/components/relational/ansi/sql/visitors/SelectionVisitor.java @@ -15,15 +15,20 @@ package org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors; import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanNode; +import org.finos.legend.engine.persistence.components.logicalplan.conditions.And; +import org.finos.legend.engine.persistence.components.logicalplan.conditions.Condition; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DerivedDataset; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Selection; import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.statements.SelectStatement; import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; import org.finos.legend.engine.persistence.components.transformer.VisitorContext; +import org.finos.legend.engine.persistence.components.util.LogicalPlanUtils; import java.util.ArrayList; import java.util.List; +import java.util.Optional; public class SelectionVisitor implements LogicalPlanVisitor { @@ -36,7 +41,30 @@ public VisitorResult visit(PhysicalPlanNode prev, Selection current, VisitorCont prev.push(selectStatement); List logicalPlanNodeList = new ArrayList<>(); - current.source().ifPresent(logicalPlanNodeList::add); + List conditions = new ArrayList<>(); + current.condition().ifPresent(conditions::add); + + if (current.source().isPresent()) + { + Dataset dataset = current.source().get(); + /* Optimize Scenarios where using Derived Dataset: + Convert unnecessary inner queries like this + select id from (select * from table where condition) + to + select id from table where condition + */ + if (dataset instanceof DerivedDataset) + { + DerivedDataset derivedDataset = (DerivedDataset) dataset; + Condition filterCondition = LogicalPlanUtils.getDatasetFilterCondition(derivedDataset); + conditions.add(filterCondition); + logicalPlanNodeList.add(derivedDataset.datasetReference()); + } + else + { + logicalPlanNodeList.add(dataset); + } + } if (current.fields().isEmpty()) { @@ -54,7 +82,11 @@ public VisitorResult visit(PhysicalPlanNode prev, Selection current, VisitorCont selectStatement.setLimit(current.limit().get()); } - current.condition().ifPresent(logicalPlanNodeList::add); + if (!conditions.isEmpty()) + { + logicalPlanNodeList.add(And.of(conditions)); + } + current.groupByFields().ifPresent(logicalPlanNodeList::addAll); current.quantifier().ifPresent(logicalPlanNodeList::add); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/nontemporal/NontemporalDeltaTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/nontemporal/NontemporalDeltaTest.java index 50f935622a9..b8d8a64037c 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/nontemporal/NontemporalDeltaTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/nontemporal/NontemporalDeltaTest.java @@ -270,11 +270,12 @@ public void verifyNontemporalDeltaWithNoVersionAndStagingFilter(GeneratorResult "sink.\"amount\" = (SELECT stage.\"amount\" FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (sink.\"digest\" <> stage.\"digest\")) AND ((stage.\"biz_date\" > '2020-01-01') AND (stage.\"biz_date\" < '2020-01-03')))," + "sink.\"biz_date\" = (SELECT stage.\"biz_date\" FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (sink.\"digest\" <> stage.\"digest\")) AND ((stage.\"biz_date\" > '2020-01-01') AND (stage.\"biz_date\" < '2020-01-03')))," + "sink.\"digest\" = (SELECT stage.\"digest\" FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (sink.\"digest\" <> stage.\"digest\")) AND ((stage.\"biz_date\" > '2020-01-01') AND (stage.\"biz_date\" < '2020-01-03'))) " + - "WHERE EXISTS (SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\" FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (sink.\"digest\" <> stage.\"digest\")) AND ((stage.\"biz_date\" > '2020-01-01') AND (stage.\"biz_date\" < '2020-01-03')))"; + "WHERE EXISTS (SELECT * FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (sink.\"digest\" <> stage.\"digest\")) AND ((stage.\"biz_date\" > '2020-01-01') AND (stage.\"biz_date\" < '2020-01-03')))"; String insertSql = "INSERT INTO \"mydb\".\"main\" (\"id\", \"name\", \"amount\", \"biz_date\", \"digest\") " + - "(SELECT * FROM (SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\" FROM \"mydb\".\"staging\" as stage WHERE (stage.\"biz_date\" > '2020-01-01') AND (stage.\"biz_date\" < '2020-01-03')) as stage " + - "WHERE NOT (EXISTS (SELECT * FROM \"mydb\".\"main\" as sink WHERE (sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\"))))"; + "(SELECT * FROM \"mydb\".\"staging\" as stage WHERE (NOT (EXISTS " + + "(SELECT * FROM \"mydb\".\"main\" as sink WHERE (sink.\"id\" = stage.\"id\") AND " + + "(sink.\"name\" = stage.\"name\")))) AND ((stage.\"biz_date\" > '2020-01-01') AND (stage.\"biz_date\" < '2020-01-03')))"; Assertions.assertEquals(AnsiTestArtifacts.expectedBaseTablePlusDigestCreateQuery, preActionsSqlList.get(0)); Assertions.assertEquals(updateSql, milestoningSqlList.get(0)); @@ -333,11 +334,12 @@ public void verifyNontemporalDeltaWithMaxVersioningNoDedupAndStagingFilters(Gene "sink.\"biz_date\" = (SELECT stage.\"biz_date\" FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (stage.\"version\" > sink.\"version\")) AND (stage.\"snapshot_id\" > 18972))," + "sink.\"digest\" = (SELECT stage.\"digest\" FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (stage.\"version\" > sink.\"version\")) AND (stage.\"snapshot_id\" > 18972))," + "sink.\"version\" = (SELECT stage.\"version\" FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (stage.\"version\" > sink.\"version\")) AND (stage.\"snapshot_id\" > 18972)) " + - "WHERE EXISTS (SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\",stage.\"version\" FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (stage.\"version\" > sink.\"version\")) AND (stage.\"snapshot_id\" > 18972))"; + "WHERE EXISTS (SELECT * FROM \"mydb\".\"staging\" as stage WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND (stage.\"version\" > sink.\"version\")) AND (stage.\"snapshot_id\" > 18972))"; String insertSql = "INSERT INTO \"mydb\".\"main\" (\"id\", \"name\", \"amount\", \"biz_date\", \"digest\", \"version\") " + - "(SELECT * FROM (SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\",stage.\"version\" FROM \"mydb\".\"staging\" as stage WHERE stage.\"snapshot_id\" > 18972) as stage " + - "WHERE NOT (EXISTS (SELECT * FROM \"mydb\".\"main\" as sink WHERE (sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\"))))"; + "(SELECT * FROM \"mydb\".\"staging\" as stage WHERE (NOT (EXISTS " + + "(SELECT * FROM \"mydb\".\"main\" as sink WHERE (sink.\"id\" = stage.\"id\") AND " + + "(sink.\"name\" = stage.\"name\")))) AND (stage.\"snapshot_id\" > 18972))"; Assertions.assertEquals(AnsiTestArtifacts.expectedBaseTablePlusDigestPlusVersionCreateQuery, preActionsSqlList.get(0)); Assertions.assertEquals(updateSql, milestoningSqlList.get(0)); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalDeltaBatchIdBasedTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalDeltaBatchIdBasedTest.java index 69b6e40427e..1f5ea08a23b 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalDeltaBatchIdBasedTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/unitemporal/UnitemporalDeltaBatchIdBasedTest.java @@ -234,19 +234,18 @@ public void verifyUnitemporalDeltaWithNoVersionAndStagingFilter(GeneratorResult String expectedMilestoneQuery = "UPDATE \"mydb\".\"main\" as sink " + "SET sink.\"batch_id_out\" = (SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN')-1 " + "WHERE (sink.\"batch_id_out\" = 999999999) AND " + - "(EXISTS (SELECT * FROM (SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\" FROM \"mydb\".\"staging\" as stage WHERE stage.\"batch_id_in\" > 5) as stage " + - "WHERE ((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND " + - "(sink.\"digest\" <> stage.\"digest\")))"; + "(EXISTS (SELECT * FROM \"mydb\".\"staging\" as stage " + + "WHERE (((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND " + + "(sink.\"digest\" <> stage.\"digest\")) AND (stage.\"batch_id_in\" > 5)))"; String expectedUpsertQuery = "INSERT INTO \"mydb\".\"main\" " + "(\"id\", \"name\", \"amount\", \"biz_date\", \"digest\", \"batch_id_in\", \"batch_id_out\") " + "(SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\"," + - "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN')," + - "999999999 " + - "FROM (SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\" FROM \"mydb\".\"staging\" as stage WHERE stage.\"batch_id_in\" > 5) as stage " + - "WHERE NOT (EXISTS (SELECT * FROM \"mydb\".\"main\" as sink " + - "WHERE (sink.\"batch_id_out\" = 999999999) " + - "AND (sink.\"digest\" = stage.\"digest\") AND ((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")))))"; + "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN'),999999999 FROM \"mydb\".\"staging\" as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM \"mydb\".\"main\" as sink WHERE (sink.\"batch_id_out\" = 999999999) " + + "AND (sink.\"digest\" = stage.\"digest\") AND ((sink.\"id\" = stage.\"id\") AND " + + "(sink.\"name\" = stage.\"name\"))))) AND (stage.\"batch_id_in\" > 5))"; Assertions.assertEquals(AnsiTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(getExpectedMetadataTableCreateQuery(), preActionsSql.get(1)); @@ -304,19 +303,18 @@ public void verifyUnitemporalDeltaWithMaxVersionNoDedupAndStagingFilter(Generato String expectedMilestoneQuery = "UPDATE \"mydb\".\"main\" as sink " + "SET sink.\"batch_id_out\" = (SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN')-1 " + "WHERE (sink.\"batch_id_out\" = 999999999) AND " + - "(EXISTS (SELECT * FROM (SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\",stage.\"version\" FROM \"mydb\".\"staging\" as stage WHERE stage.\"batch_id_in\" > 5) as stage " + - "WHERE ((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND " + - "(stage.\"version\" > sink.\"version\")))"; + "(EXISTS (SELECT * FROM \"mydb\".\"staging\" as stage WHERE " + + "(((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")) AND " + + "(stage.\"version\" > sink.\"version\")) AND (stage.\"batch_id_in\" > 5)))"; String expectedUpsertQuery = "INSERT INTO \"mydb\".\"main\" " + "(\"id\", \"name\", \"amount\", \"biz_date\", \"digest\", \"version\", \"batch_id_in\", \"batch_id_out\") " + "(SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\",stage.\"version\"," + - "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN')," + - "999999999 " + - "FROM (SELECT stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\",stage.\"version\" FROM \"mydb\".\"staging\" as stage WHERE stage.\"batch_id_in\" > 5) as stage " + - "WHERE NOT (EXISTS (SELECT * FROM \"mydb\".\"main\" as sink " + - "WHERE (sink.\"batch_id_out\" = 999999999) " + - "AND (stage.\"version\" <= sink.\"version\") AND ((sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\")))))"; + "(SELECT COALESCE(MAX(batch_metadata.\"table_batch_id\"),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.\"table_name\") = 'MAIN'),999999999 FROM \"mydb\".\"staging\" as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM \"mydb\".\"main\" as sink WHERE (sink.\"batch_id_out\" = 999999999) " + + "AND (stage.\"version\" <= sink.\"version\") AND ((sink.\"id\" = stage.\"id\") " + + "AND (sink.\"name\" = stage.\"name\"))))) AND (stage.\"batch_id_in\" > 5))"; Assertions.assertEquals(AnsiTestArtifacts.expectedMainTableBatchIdAndVersionBasedCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(getExpectedMetadataTableCreateQuery(), preActionsSql.get(1)); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtilsTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtilsTest.java index 443fd6b3e1f..afc8c688070 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtilsTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-ansi/src/test/java/org/finos/legend/engine/persistence/components/util/LogicalPlanUtilsTest.java @@ -17,7 +17,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.finos.legend.engine.persistence.components.IngestModeTest; -import org.finos.legend.engine.persistence.components.ingestmode.deduplication.DatasetFilterAndDeduplicator; +import org.finos.legend.engine.persistence.components.ingestmode.deduplication.DatasetDeduplicator; import org.finos.legend.engine.persistence.components.ingestmode.deduplication.MaxVersionStrategy; import org.finos.legend.engine.persistence.components.ingestmode.deduplication.VersioningStrategy; import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; @@ -58,7 +58,7 @@ public void testDeduplicateByMaxVersion() List primaryKeys = Arrays.asList("id", "name"); VersioningStrategy versioningStrategy = MaxVersionStrategy.builder().versioningField("version").performDeduplication(true).versioningComparator(GREATER_THAN).build(); - Selection selection = (Selection) versioningStrategy.accept(new DatasetFilterAndDeduplicator(dataset, primaryKeys)); + Selection selection = (Selection) versioningStrategy.accept(new DatasetDeduplicator(dataset, primaryKeys)); LogicalPlan logicalPlan = LogicalPlan.builder().addOps(selection).build(); SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); List list = physicalPlan.getSqlList(); @@ -88,7 +88,7 @@ public void testDeduplicateByMaxVersionAndFilterDataset() .build(); VersioningStrategy versioningStrategy = MaxVersionStrategy.builder().versioningField("version").performDeduplication(true).versioningComparator(GREATER_THAN).build(); - Selection selection = (Selection) versioningStrategy.accept(new DatasetFilterAndDeduplicator(dataset, primaryKeys)); + Selection selection = (Selection) versioningStrategy.accept(new DatasetDeduplicator(dataset, primaryKeys)); LogicalPlan logicalPlan = LogicalPlan.builder().addOps(selection).build(); SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/pom.xml b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/pom.xml new file mode 100644 index 00000000000..6615e27de00 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/pom.xml @@ -0,0 +1,128 @@ + + + + + org.finos.legend.engine + legend-engine-xt-persistence-component + 4.17.1-SNAPSHOT + + 4.0.0 + + legend-engine-xt-persistence-component-relational-bigquery + jar + Legend Engine - XT - Persistence - Component - Relational Bigquery + + + + + com.google.cloud + libraries-bom + 26.14.0 + pom + import + + + + + + + org.finos.legend.engine + legend-engine-xt-persistence-component-logical-plan + + + org.finos.legend.engine + legend-engine-xt-persistence-component-physical-plan + + + org.finos.legend.engine + legend-engine-xt-persistence-component-relational-core + + + org.finos.legend.engine + legend-engine-xt-persistence-component-relational-ansi + + + + + com.google.cloud + google-cloud-bigquery + provided + + + commons-logging + commons-logging + + + + + com.google.cloud + google-cloud-core + provided + + + com.google.auth + google-auth-library-oauth2-http + provided + + + org.slf4j + slf4j-api + + + + + org.finos.legend.engine + legend-engine-xt-persistence-component-relational-test + test-jar + + + org.finos.legend.engine + legend-engine-xt-persistence-component-relational-ansi + test-jar + + + org.junit.jupiter + junit-jupiter-api + + + org.junit.jupiter + junit-jupiter-engine + + + com.opencsv + opencsv + 4.6 + test + + + org.apache.commons + commons-lang3 + + + commons-logging + commons-logging + + + + + org.apache.commons + commons-lang3 + 3.10 + test + + + diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/BigQuerySink.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/BigQuerySink.java new file mode 100644 index 00000000000..b2e697458e8 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/BigQuerySink.java @@ -0,0 +1,231 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery; + +import org.finos.legend.engine.persistence.components.executor.Executor; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.ClusterKey; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.PartitionKey; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Alter; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Create; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Delete; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Truncate; +import org.finos.legend.engine.persistence.components.logicalplan.values.BatchEndTimestamp; +import org.finos.legend.engine.persistence.components.logicalplan.values.BatchStartTimestamp; +import org.finos.legend.engine.persistence.components.optimizer.Optimizer; +import org.finos.legend.engine.persistence.components.relational.CaseConversion; +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.ansi.AnsiSqlSink; +import org.finos.legend.engine.persistence.components.relational.api.RelationalConnection; +import org.finos.legend.engine.persistence.components.relational.bigquery.executor.BigQueryConnection; +import org.finos.legend.engine.persistence.components.relational.bigquery.executor.BigQueryExecutor; +import org.finos.legend.engine.persistence.components.relational.bigquery.executor.BigQueryHelper; +import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.LowerCaseOptimizer; +import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.UpperCaseOptimizer; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.BigQueryDataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.BigQueryDataTypeToLogicalDataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.AlterVisitor; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.BatchEndTimestampVisitor; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.BatchStartTimestampVisitor; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.ClusterKeyVisitor; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.DeleteVisitor; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.FieldVisitor; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.PartitionKeyVisitor; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.SQLCreateVisitor; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.SchemaDefinitionVisitor; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor.TruncateVisitor; +import org.finos.legend.engine.persistence.components.relational.sql.TabularData; +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; +import org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.util.Capability; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class BigQuerySink extends AnsiSqlSink +{ + private static final RelationalSink INSTANCE; + private static final Set CAPABILITIES; + private static final Map, LogicalPlanVisitor> LOGICAL_PLAN_VISITOR_BY_CLASS; + private static final Map> IMPLICIT_DATA_TYPE_MAPPING; + private static final Map> EXPLICIT_DATA_TYPE_MAPPING; + + static + { + Set capabilities = new HashSet<>(); + capabilities.add(Capability.MERGE); + capabilities.add(Capability.ADD_COLUMN); + capabilities.add(Capability.IMPLICIT_DATA_TYPE_CONVERSION); + capabilities.add(Capability.EXPLICIT_DATA_TYPE_CONVERSION); + capabilities.add(Capability.DATA_TYPE_LENGTH_CHANGE); + capabilities.add(Capability.DATA_TYPE_SCALE_CHANGE); + CAPABILITIES = Collections.unmodifiableSet(capabilities); + + Map, LogicalPlanVisitor> logicalPlanVisitorByClass = new HashMap<>(); + logicalPlanVisitorByClass.put(SchemaDefinition.class, new SchemaDefinitionVisitor()); + logicalPlanVisitorByClass.put(Create.class, new SQLCreateVisitor()); + logicalPlanVisitorByClass.put(ClusterKey.class, new ClusterKeyVisitor()); + logicalPlanVisitorByClass.put(PartitionKey.class, new PartitionKeyVisitor()); + logicalPlanVisitorByClass.put(Alter.class, new AlterVisitor()); + logicalPlanVisitorByClass.put(Delete.class, new DeleteVisitor()); + logicalPlanVisitorByClass.put(Field.class, new FieldVisitor()); + logicalPlanVisitorByClass.put(Truncate.class, new TruncateVisitor()); + logicalPlanVisitorByClass.put(BatchEndTimestamp.class, new BatchEndTimestampVisitor()); + logicalPlanVisitorByClass.put(BatchStartTimestamp.class, new BatchStartTimestampVisitor()); + LOGICAL_PLAN_VISITOR_BY_CLASS = Collections.unmodifiableMap(logicalPlanVisitorByClass); + + Map> implicitDataTypeMapping = new HashMap<>(); + implicitDataTypeMapping.put(DataType.INTEGER, getUnmodifiableDataTypesSet(DataType.INT, DataType.BIGINT, DataType.TINYINT, DataType.SMALLINT, DataType.INT64)); + implicitDataTypeMapping.put(DataType.NUMERIC, getUnmodifiableDataTypesSet(DataType.NUMERIC, DataType.NUMBER, DataType.DECIMAL, DataType.INT, DataType.INTEGER, DataType.BIGINT, DataType.TINYINT, DataType.SMALLINT, DataType.INT64)); + implicitDataTypeMapping.put(DataType.FLOAT, getUnmodifiableDataTypesSet(DataType.REAL, DataType.DOUBLE, DataType.FLOAT64, DataType.INT, DataType.INTEGER, DataType.BIGINT, DataType.TINYINT, DataType.SMALLINT, DataType.INT64, DataType.NUMBER, DataType.NUMERIC, DataType.DECIMAL)); + implicitDataTypeMapping.put(DataType.STRING, getUnmodifiableDataTypesSet(DataType.STRING, DataType.CHAR, DataType.CHARACTER, DataType.VARCHAR, DataType.LONGNVARCHAR, DataType.LONGTEXT, DataType.TEXT)); + implicitDataTypeMapping.put(DataType.DATETIME, Collections.singleton(DataType.DATE)); + IMPLICIT_DATA_TYPE_MAPPING = Collections.unmodifiableMap(implicitDataTypeMapping); + + Map> explicitDataTypeMapping = new HashMap<>(); + explicitDataTypeMapping.put(DataType.INT, getUnmodifiableDataTypesSet(DataType.NUMBER, DataType.NUMERIC, DataType.DECIMAL, DataType.REAL, DataType.FLOAT, DataType.DOUBLE, DataType.FLOAT64)); + explicitDataTypeMapping.put(DataType.INTEGER, getUnmodifiableDataTypesSet(DataType.NUMBER, DataType.NUMERIC, DataType.DECIMAL, DataType.REAL, DataType.FLOAT, DataType.DOUBLE, DataType.FLOAT64)); + explicitDataTypeMapping.put(DataType.BIGINT, getUnmodifiableDataTypesSet(DataType.NUMBER, DataType.NUMERIC, DataType.DECIMAL, DataType.REAL, DataType.FLOAT, DataType.DOUBLE, DataType.FLOAT64)); + explicitDataTypeMapping.put(DataType.TINYINT, getUnmodifiableDataTypesSet(DataType.NUMBER, DataType.NUMERIC, DataType.DECIMAL, DataType.REAL, DataType.FLOAT, DataType.DOUBLE, DataType.FLOAT64)); + explicitDataTypeMapping.put(DataType.SMALLINT, getUnmodifiableDataTypesSet(DataType.NUMBER, DataType.NUMERIC, DataType.DECIMAL, DataType.REAL, DataType.FLOAT, DataType.DOUBLE, DataType.FLOAT64)); + explicitDataTypeMapping.put(DataType.INT64, getUnmodifiableDataTypesSet(DataType.NUMBER, DataType.NUMERIC, DataType.DECIMAL, DataType.REAL, DataType.FLOAT, DataType.DOUBLE, DataType.FLOAT64)); + explicitDataTypeMapping.put(DataType.NUMBER, getUnmodifiableDataTypesSet(DataType.REAL, DataType.FLOAT, DataType.DOUBLE, DataType.FLOAT64)); + explicitDataTypeMapping.put(DataType.NUMERIC, getUnmodifiableDataTypesSet(DataType.REAL, DataType.FLOAT, DataType.DOUBLE, DataType.FLOAT64)); + explicitDataTypeMapping.put(DataType.DECIMAL, getUnmodifiableDataTypesSet(DataType.REAL, DataType.FLOAT, DataType.DOUBLE, DataType.FLOAT64)); + EXPLICIT_DATA_TYPE_MAPPING = Collections.unmodifiableMap(explicitDataTypeMapping); + INSTANCE = new BigQuerySink(); + } + + private static Set getUnmodifiableDataTypesSet(DataType... dataTypes) + { + return Collections.unmodifiableSet(new HashSet<>(Arrays.asList(dataTypes))); + } + + public static RelationalSink get() + { + return INSTANCE; + } + + private BigQuerySink() + { + super( + CAPABILITIES, + IMPLICIT_DATA_TYPE_MAPPING, + EXPLICIT_DATA_TYPE_MAPPING, + SqlGenUtils.BACK_QUOTE_IDENTIFIER, + LOGICAL_PLAN_VISITOR_BY_CLASS, + (executor, sink, dataset) -> sink.doesTableExist(dataset), + (executor, sink, dataset) -> sink.validateDatasetSchema(dataset, new BigQueryDataTypeMapping()), + (executor, sink, tableName, schemaName, databaseName) -> sink.constructDatasetFromDatabase(tableName, schemaName, databaseName, new BigQueryDataTypeToLogicalDataTypeMapping())); + } + + @Override + public Optional optimizerForCaseConversion(CaseConversion caseConversion) + { + switch (caseConversion) + { + case TO_LOWER: + return Optional.of(new LowerCaseOptimizer()); + case TO_UPPER: + return Optional.of(new UpperCaseOptimizer()); + case NONE: + return Optional.empty(); + default: + throw new IllegalArgumentException("Unrecognized case conversion: " + caseConversion); + } + } + + @Override + public Executor getRelationalExecutor(RelationalConnection relationalConnection) + { + if (relationalConnection instanceof BigQueryConnection) + { + BigQueryConnection bigQueryConnection = (BigQueryConnection) relationalConnection; + return new BigQueryExecutor(this, BigQueryHelper.of(bigQueryConnection.bigQuery())); + } + else + { + throw new UnsupportedOperationException("Only BigQueryConnection is supported for BigQuery Sink"); + } + } + + //evolve to = field to replace main column (datatype) + //evolve from = reference field to compare sizing/nullability requirements + @Override + public Field evolveFieldLength(Field evolveFrom, Field evolveTo) + { + Optional oldScale = evolveFrom.type().scale(); + Optional newScale = evolveTo.type().scale(); + Optional scale = getMaximumValue(oldScale, newScale, true); + + Optional oldLength = evolveFrom.type().length(); + Optional newLength = evolveTo.type().length(); + Optional length; + + if (scale.isPresent()) + { + //if scale is present, by design, precision is also present + Optional oldIntegralLength = oldScale.map(scaleValue -> Optional.of(oldLength.get() - scaleValue)).orElse(oldLength); + Optional newIntegralLength = newScale.map(scaleValue -> Optional.of(newLength.get() - scaleValue)).orElse(newLength); + Optional integralLength = getMaximumValue(oldIntegralLength, newIntegralLength, false); + length = integralLength.map(integralLengthValue -> integralLengthValue + scale.get()); + } + else + { + length = getMaximumValue(oldLength, newLength, false); + } + + return this.createNewField(evolveTo, evolveFrom, length, scale); + } + + private static Optional getMaximumValue(Optional oldValue, Optional newValue, boolean classifyEmptyValueAsZero) + { + Optional value = newValue; + if (oldValue.isPresent() && newValue.isPresent()) + { + if (newValue.get() <= oldValue.get()) + { + value = oldValue; + } + } + else if (!classifyEmptyValueAsZero && !oldValue.isPresent() || classifyEmptyValueAsZero && oldValue.isPresent()) + { + value = oldValue; + } + return value; + } + + @Override + public Field createNewField(Field evolveTo, Field evolveFrom, Optional length, Optional scale) + { + FieldType modifiedFieldType = length.isPresent() ? FieldType.of(evolveTo.type().dataType(), length, scale) : FieldType.of(evolveTo.type().dataType(), Optional.empty(), Optional.empty()); + boolean nullability = evolveTo.nullable() || evolveFrom.nullable(); + + return Field.builder().name(evolveTo.name()).primaryKey(evolveTo.primaryKey()) + .fieldAlias(evolveTo.fieldAlias()).nullable(nullability) + .identity(evolveTo.identity()).unique(evolveTo.unique()) + .defaultValue(evolveTo.defaultValue()).type(modifiedFieldType).build(); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryConnection.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryConnection.java new file mode 100644 index 00000000000..afc9f18e0d8 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryConnection.java @@ -0,0 +1,38 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.executor; + +import com.google.cloud.bigquery.BigQuery; +import org.finos.legend.engine.persistence.components.relational.api.RelationalConnection; + +public class BigQueryConnection implements RelationalConnection +{ + private final BigQuery bigQuery; + + public BigQuery bigQuery() + { + return bigQuery; + } + + private BigQueryConnection(BigQuery bigQuery) + { + this.bigQuery = bigQuery; + } + + public static BigQueryConnection of(BigQuery bigQuery) + { + return new BigQueryConnection(bigQuery); + } +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryExecutor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryExecutor.java new file mode 100644 index 00000000000..8d1014ba648 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryExecutor.java @@ -0,0 +1,146 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.executor; + +import org.finos.legend.engine.persistence.components.executor.Executor; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.relational.sql.TabularData; +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.statements.DDLStatement; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; + +public class BigQueryExecutor implements Executor +{ + private final BigQuerySink bigQuerySink; + private final BigQueryHelper bigQueryHelper; + + public BigQueryExecutor(BigQuerySink bigQuerySink, BigQueryHelper bigQueryHelper) + { + this.bigQuerySink = bigQuerySink; + this.bigQueryHelper = bigQueryHelper; + } + + @Override + public void executePhysicalPlan(SqlPlan physicalPlan) + { + executePhysicalPlan(physicalPlan, new HashMap<>()); + } + + @Override + public void executePhysicalPlan(SqlPlan physicalPlan, Map placeholderKeyValues) + { + boolean containsDDLStatements = physicalPlan.ops().stream().anyMatch(DDLStatement.class::isInstance); + List sqlList = physicalPlan.getSqlList(); + + if (containsDDLStatements) + { + for (String sql : sqlList) + { + String enrichedSql = getEnrichedSql(placeholderKeyValues, sql); + bigQueryHelper.executeQuery(enrichedSql); + } + } + else + { + for (String sql : sqlList) + { + String enrichedSql = getEnrichedSql(placeholderKeyValues, sql); + bigQueryHelper.executeStatement(enrichedSql); + } + } + } + + @Override + public List executePhysicalPlanAndGetResults(SqlPlan physicalPlan) + { + return executePhysicalPlanAndGetResults(physicalPlan, new HashMap<>()); + } + + @Override + public List executePhysicalPlanAndGetResults(SqlPlan physicalPlan, Map placeholderKeyValues) + { + List resultSetList = new ArrayList<>(); + for (String sql : physicalPlan.getSqlList()) + { + String enrichedSql = getEnrichedSql(placeholderKeyValues, sql); + List> queryResult = bigQueryHelper.executeQuery(enrichedSql); + if (!queryResult.isEmpty()) + { + resultSetList.add(new TabularData(queryResult)); + } + } + return resultSetList; + } + + @Override + public boolean datasetExists(Dataset dataset) + { + return bigQuerySink.datasetExistsFn().apply(this, bigQueryHelper, dataset); + } + + @Override + public void validateMainDatasetSchema(Dataset dataset) + { + bigQuerySink.validateMainDatasetSchemaFn().execute(this, bigQueryHelper, dataset); + } + + @Override + public Dataset constructDatasetFromDatabase(String tableName, String schemaName, String databaseName) + { + return bigQuerySink.constructDatasetFromDatabaseFn().execute(this, bigQueryHelper, tableName, schemaName, databaseName); + } + + @Override + public void begin() + { + bigQueryHelper.beginTransaction(); + } + + @Override + public void commit() + { + bigQueryHelper.commitTransaction(); + } + + @Override + public void revert() + { + bigQueryHelper.revertTransaction(); + } + + @Override + public void close() + { + bigQueryHelper.close(); + } + + private String getEnrichedSql(Map placeholderKeyValues, String sql) + { + String enrichedSql = sql; + for (Map.Entry entry : placeholderKeyValues.entrySet()) + { + enrichedSql = enrichedSql.replaceAll(Pattern.quote(entry.getKey()), entry.getValue()); + } + return enrichedSql; + } + +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryHelper.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryHelper.java new file mode 100644 index 00000000000..8e7168db320 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryHelper.java @@ -0,0 +1,418 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.executor; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.TableId; +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; +import org.finos.legend.engine.persistence.components.logicalplan.conditions.And; +import org.finos.legend.engine.persistence.components.logicalplan.conditions.Equals; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetReferenceImpl; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Selection; +import org.finos.legend.engine.persistence.components.logicalplan.values.FieldValue; +import org.finos.legend.engine.persistence.components.logicalplan.values.StringValue; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.constraints.columns.PKColumnConstraint; +import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutionHelper; +import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.sql.JdbcPropertiesToLogicalDataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.ColumnConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.NotNullColumnConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.DataType; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.Column; +import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +public class BigQueryHelper implements RelationalExecutionHelper +{ + public static final String PRIMARY_KEY_INFO_TABLE_NAME = "INFORMATION_SCHEMA.KEY_COLUMN_USAGE"; + private static final Logger LOGGER = LoggerFactory.getLogger(BigQueryHelper.class); + private static final String TABLE_NAME = "TABLE_NAME"; + private static final String TABLE_SCHEMA = "TABLE_SCHEMA"; + private static final String CONSTRAINT_NAME = "CONSTRAINT_NAME"; + private static final String CONSTRAINT_NAME_QUANTIFIER_PK = ".pk$"; + private static final Function CONSTRAINT_NAME_PROVIDER_PK = tableName -> tableName + CONSTRAINT_NAME_QUANTIFIER_PK; + + private final BigQuery bigQuery; + private BigQueryTransactionManager transactionManager; + + public static BigQueryHelper of(BigQuery bigQuery) + { + if (bigQuery != null) + { + return new BigQueryHelper(bigQuery); + } + throw new RuntimeException("Sink initialized without connection can only be used for SQL generation APIs, but used with ingestion API"); + } + + private BigQueryHelper(BigQuery bigQuery) + { + this.bigQuery = bigQuery; + } + + public void beginTransaction() + { + try + { + this.transactionManager = new BigQueryTransactionManager(bigQuery); + this.transactionManager.beginTransaction(); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + public void commitTransaction() + { + if (this.transactionManager != null) + { + try + { + this.transactionManager.commitTransaction(); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + } + + public void revertTransaction() + { + if (this.transactionManager != null) + { + try + { + this.transactionManager.revertTransaction(); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + } + + public void closeTransactionManager() + { + if (this.transactionManager != null) + { + try + { + this.transactionManager.close(); + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + finally + { + this.transactionManager = null; + } + } + } + + public boolean doesTableExist(Dataset dataset) + { + String projectName = dataset.datasetReference().database().orElse(null); + String datasetName = dataset.datasetReference().group().orElseThrow(IllegalStateException::new); + String tableName = dataset.datasetReference().name().orElseThrow(IllegalStateException::new); + + TableId tableId = projectName == null ? + TableId.of(datasetName, tableName) : + TableId.of(projectName, datasetName, tableName); + + com.google.cloud.bigquery.Table table = this.bigQuery.getTable(tableId); + boolean tableExists = table != null && table.exists(); + return tableExists; + } + + public void validateDatasetSchema(Dataset dataset, DataTypeMapping datatypeMapping) + { + String name = dataset.datasetReference().name().orElseThrow(IllegalStateException::new); + String schema = dataset.datasetReference().group().orElse(null); + + com.google.cloud.bigquery.Table table = this.bigQuery.getTable(TableId.of(schema, name)); + List primaryKeysInDb = this.fetchPrimaryKeys(name, schema, dataset.datasetReference().database().orElse(null)); + FieldList dbFields = table.getDefinition().getSchema().getFields(); + List userFields = new ArrayList<>(dataset.schema().fields()); + List userColumns = convertUserProvidedFieldsToColumns(userFields, datatypeMapping); + List dbColumns = new ArrayList<>(); + + for (com.google.cloud.bigquery.Field dbField : dbFields) + { + String columnName = dbField.getName(); + + // Get the datatype + String typeName = dbField.getType().getStandardType().name(); + Integer columnSize = Objects.nonNull(dbField.getMaxLength()) ? Integer.valueOf(dbField.getMaxLength().intValue()) : Objects.nonNull(dbField.getPrecision()) ? Integer.valueOf(dbField.getPrecision().intValue()) : null; + Integer scale = Objects.nonNull(dbField.getScale()) ? Integer.valueOf(dbField.getScale().intValue()) : null; + + org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType matchedDataType = org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType.valueOf(typeName.toUpperCase()); + + FieldType fieldType = FieldType.of(matchedDataType, columnSize, scale); + DataType physicalDataType = datatypeMapping.getDataType(fieldType); + + // Check the constraints + List columnConstraints = new ArrayList<>(); + if (com.google.cloud.bigquery.Field.Mode.REQUIRED.equals(dbField.getMode())) + { + columnConstraints.add(new NotNullColumnConstraint()); + } + if (primaryKeysInDb.contains(columnName)) + { + columnConstraints.add(new PKColumnConstraint()); + } + + Column column = new Column(columnName, physicalDataType, columnConstraints, null); + dbColumns.add(column); + } + + // Compare the schemas + validateColumns(userColumns, dbColumns); + } + + public Dataset constructDatasetFromDatabase(String tableName, String schemaName, String databaseName, JdbcPropertiesToLogicalDataTypeMapping mapping) + { + List primaryKeysInDb = this.fetchPrimaryKeys(tableName, schemaName, databaseName); + com.google.cloud.bigquery.Table table = this.bigQuery.getTable(TableId.of(schemaName, tableName)); + + // Get all columns + List fields = new ArrayList<>(); + for (com.google.cloud.bigquery.Field dbField : table.getDefinition().getSchema().getFields()) + { + String columnName = dbField.getName(); + String typeName = dbField.getType().getStandardType().name(); + String dataType = dbField.getType().name(); + Integer columnSize = Objects.nonNull(dbField.getMaxLength()) ? Integer.valueOf(dbField.getMaxLength().intValue()) : Objects.nonNull(dbField.getPrecision()) ? Integer.valueOf(dbField.getPrecision().intValue()) : null; + Integer decimalDigits = Objects.nonNull(dbField.getScale()) ? Integer.valueOf(dbField.getScale().intValue()) : null; + + // Construct type + FieldType fieldType = mapping.getDataType(typeName.toUpperCase(), dataType.toUpperCase(), columnSize, decimalDigits); + + // Construct constraints + boolean nullable = !com.google.cloud.bigquery.Field.Mode.REQUIRED.equals(dbField.getMode()); + boolean primaryKey = primaryKeysInDb.contains(columnName); + + Field field = Field.builder() + .name(columnName) + .type(fieldType) + .nullable(nullable) + .primaryKey(primaryKey) + .build(); + + fields.add(field); + } + + SchemaDefinition schemaDefinition = SchemaDefinition.builder() + .addAllFields(fields) + .build(); + return DatasetDefinition.builder().name(tableName).database(databaseName).group(schemaName).schema(schemaDefinition).build(); + } + + private List fetchPrimaryKeys(String tableName, String schemaName, String databaseName) + { + RelationalTransformer relationalTransformer = new RelationalTransformer(BigQuerySink.get()); + LogicalPlan logicalPlanFetchPrimaryKeys = getLogicalPlanFetchPrimaryKeys(tableName, schemaName, databaseName); + SqlPlan sqlPlan = relationalTransformer.generatePhysicalPlan(logicalPlanFetchPrimaryKeys); + List> resultSet = executeQuery(sqlPlan.getSql()); + return resultSet.stream().map(resultEntry -> (String) resultEntry.get(COLUMN_NAME)).collect(Collectors.toList()); + } + + private static LogicalPlan getLogicalPlanFetchPrimaryKeys(String tableName, String schemaName, String databaseName) + { + return LogicalPlan.builder().addOps(Selection.builder() + .addFields(FieldValue.builder().fieldName(COLUMN_NAME).build()) + .source(DatasetReferenceImpl.builder() + .database(databaseName) + .group(schemaName) + .name(PRIMARY_KEY_INFO_TABLE_NAME).build()) + .condition(And.of(Arrays.asList( + Equals.of(FieldValue.builder().fieldName(TABLE_SCHEMA).build(), + StringValue.of(schemaName)), + Equals.of(FieldValue.builder().fieldName(TABLE_NAME).build(), + StringValue.of(tableName)), + Equals.of(FieldValue.builder().fieldName(CONSTRAINT_NAME).build(), + StringValue.of(CONSTRAINT_NAME_PROVIDER_PK.apply(tableName)))))) + .build()).build(); + } + + public static void validateColumns(List userColumns, List dbColumns) + { + if (userColumns.size() != dbColumns.size()) + { + throw new IllegalStateException("Number of columns in user-provided schema doesn't match with the schema in the database"); + } + for (Column userColumn : userColumns) + { + Column matchedColumn = dbColumns.stream().filter(dbColumn -> dbColumn.getColumnName().equals(userColumn.getColumnName())).findFirst().orElseThrow(() -> new IllegalStateException("Column in user-provided schema doesn't match any column in the schema in the database")); + if (!userColumn.equals(matchedColumn)) + { + throw new IllegalStateException("Column in user-provided schema doesn't match the corresponding column in the schema in the database"); + } + } + } + + public static List convertUserProvidedFieldsToColumns(List userFields, DataTypeMapping datatypeMapping) + { + List columnList = new ArrayList<>(); + + for (Field f : userFields) + { + DataType dataType = datatypeMapping.getDataType(f.type()); + List columnConstraints = new ArrayList<>(); + if (!f.nullable() || f.primaryKey()) + { + columnConstraints.add(new NotNullColumnConstraint()); + } + if (f.primaryKey()) + { + columnConstraints.add(new PKColumnConstraint()); + } + Column column = new Column(f.name(), dataType, columnConstraints, null); + columnList.add(column); + } + + return columnList; + } + + + public void executeStatement(String sql) + { + List sqls = Collections.singletonList(sql); + executeStatements(sqls); + } + + // Execute statements in a transaction - either use an existing one or use a new one + public void executeStatements(List sqls) + { + if (this.transactionManager != null) + { + try + { + for (String sql : sqls) + { + this.transactionManager.executeInCurrentTransaction(sql); + } + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + else + { + BigQueryTransactionManager txManager = null; + try + { + txManager = new BigQueryTransactionManager(bigQuery); + txManager.beginTransaction(); + for (String sql : sqls) + { + txManager.executeInCurrentTransaction(sql); + } + txManager.commitTransaction(); + } + catch (Exception e) + { + LOGGER.error("Error executing SQL statements: " + sqls, e); + if (txManager != null) + { + try + { + txManager.revertTransaction(); + } + catch (InterruptedException e2) + { + throw new RuntimeException(e2); + } + } + throw new RuntimeException(e); + } + finally + { + if (txManager != null) + { + try + { + txManager.close(); + } + catch (InterruptedException e) + { + LOGGER.error("Error closing transaction manager.", e); + } + } + } + } + } + + public List> executeQuery(String sql) + { + if (this.transactionManager != null) + { + return this.transactionManager.convertResultSetToList(sql); + } + else + { + BigQueryTransactionManager txManager = null; + try + { + txManager = new BigQueryTransactionManager(bigQuery); + return txManager.convertResultSetToList(sql); + } + catch (Exception e) + { + throw new RuntimeException("Error executing SQL query: " + sql, e); + } + finally + { + if (txManager != null) + { + try + { + txManager.close(); + } + catch (InterruptedException e) + { + LOGGER.error("Error closing transaction manager.", e); + } + } + } + } + } + + @Override + public void close() + { + closeTransactionManager(); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryTransactionManager.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryTransactionManager.java new file mode 100644 index 00000000000..7db1a4c5ad8 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryTransactionManager.java @@ -0,0 +1,179 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.executor; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.ConnectionProperty; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldValue; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.LegacySQLTypeName; +import com.google.cloud.bigquery.QueryJobConfiguration; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +public class BigQueryTransactionManager +{ + private final BigQuery bigQuery; + private String sessionId; + private static final String CONNECTION_SESSION_PROPERTY = "session_id"; + + public BigQueryTransactionManager(BigQuery bigQuery) + { + this.bigQuery = bigQuery; + } + + public void close() throws InterruptedException + { + if (this.sessionId != null) + { + try + { + executeSql("CALL BQ.ABORT_SESSION();"); + } + finally + { + this.sessionId = null; + } + } + } + + public void beginTransaction() throws InterruptedException + { + Job job = this.bigQuery.create(JobInfo.of(QueryJobConfiguration + .newBuilder("BEGIN TRANSACTION") + .setCreateSession(true) + .build())); + job.waitFor(); + this.sessionId = job.getStatistics().getSessionInfo().getSessionId(); + } + + public void commitTransaction() throws InterruptedException + { + if (this.sessionId != null) + { + executeSql("COMMIT TRANSACTION"); + } + else + { + throw new IllegalStateException("No Transaction started, nothing to commit"); + } + } + + public void revertTransaction() throws InterruptedException + { + if (this.sessionId != null) + { + executeSql("ROLLBACK TRANSACTION"); + } + else + { + throw new IllegalStateException("No Transaction started, nothing to revert"); + } + } + + public boolean executeInCurrentTransaction(String sql) throws InterruptedException + { + Job job = this.executeSql(sql); + return job.getStatus().getError() == null; + } + + public List> convertResultSetToList(String sql) + { + try + { + List> resultList = new ArrayList<>(); + Job job = this.executeSql(sql); + for (FieldValueList fieldValues : job.getQueryResults().getValues()) + { + Map row = new HashMap<>(); + for (Field field: job.getQueryResults().getSchema().getFields()) + { + FieldValue value = fieldValues.get(field.getName()); + Object objectValue; + switch (field.getType().name()) + { + case "BYTES": + objectValue = value.getBytesValue(); + break; + case "STRING": + objectValue = value.getStringValue(); + break; + case "INTEGER": + objectValue = value.getLongValue(); + break; + case "FLOAT": + objectValue = value.getDoubleValue(); + break; + case "NUMERIC": + case "BIGNUMERIC": + objectValue = value.getNumericValue(); + break; + case "BOOLEAN": + objectValue = value.getBooleanValue(); + break; + case "TIMESTAMP": + objectValue = value.getTimestampInstant(); + break; + case "RECORD": + objectValue = value.getRecordValue(); + break; + default: + objectValue = value.getValue(); + } + String key = field.getName(); + row.put(key, objectValue); + } + resultList.add(row); + } + return resultList; + } + catch (InterruptedException e) + { + throw new RuntimeException(e); + } + } + + private Job executeSql(String sqlQuery) throws InterruptedException + { + JobId jobId = JobId.of(UUID.randomUUID().toString()); + Job job = this.bigQuery.create(JobInfo.newBuilder(getQueryJobConfiguration(sqlQuery)).setJobId(jobId).build()); + return job.waitFor(); + } + + private QueryJobConfiguration getQueryJobConfiguration(String sqlQuery) + { + if (this.sessionId == null) + { + return QueryJobConfiguration.newBuilder(sqlQuery).setUseLegacySql(false).build(); + } + return QueryJobConfiguration + .newBuilder(sqlQuery) + .setUseLegacySql(false) + .setCreateSession(false) + .setConnectionProperties(Arrays.asList(ConnectionProperty.newBuilder() + .setKey(CONNECTION_SESSION_PROPERTY) + .setValue(this.sessionId) + .build())).build(); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/BigQueryDataTypeMapping.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/BigQueryDataTypeMapping.java new file mode 100644 index 00000000000..f390abe2d9e --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/BigQueryDataTypeMapping.java @@ -0,0 +1,121 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql; + +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema.Bool; +import org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema.Bytes; +import org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema.Float64; +import org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema.Json; +import org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema.String; +import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.DataType; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.Date; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.DateTime; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.Int64; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.Numeric; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.Time; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.Timestamp; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.VariableSizeDataType; + + +public class BigQueryDataTypeMapping implements DataTypeMapping +{ + public DataType getDataType(FieldType type) + { + VariableSizeDataType dataType; + switch (type.dataType()) + { + // Numeric Data Types + case INT: + case INTEGER: + case BIGINT: + case TINYINT: + case SMALLINT: + case INT64: + dataType = new Int64(); + break; + case NUMBER: + case NUMERIC: + case DECIMAL: + dataType = new Numeric(); + type.length().ifPresent(dataType::setLength); + type.scale().ifPresent(dataType::setScale); + break; + case REAL: + case FLOAT: + case DOUBLE: + case FLOAT64: + dataType = new Float64(); + break; + // String & Binary types + case CHAR: + case CHARACTER: + case VARCHAR: + case LONGVARCHAR: + case LONGTEXT: + case TEXT: + case STRING: + dataType = new String(); + type.length().ifPresent(dataType::setLength); + break; + case BINARY: + case VARBINARY: + case LONGVARBINARY: + case BYTES: + dataType = new Bytes(); + type.length().ifPresent(dataType::setLength); + break; + // Date & Time types + case DATE: + dataType = new Date(); + break; + case TIME: + dataType = new Time(); + break; + case DATETIME: + dataType = new DateTime(); + break; + case TIMESTAMP: + dataType = new Timestamp(); + break; + // Other types + case BOOLEAN: + case BOOL: + dataType = new Bool(); + break; + case JSON: + dataType = new Json(); + break; + case NCHAR: + case NVARCHAR: + case LONGNVARCHAR: + case BIT: + case TIMESTAMP_NTZ: + case TIMESTAMP_TZ: + case TIMESTAMP_LTZ: + case TIMESTAMPTZ: + case VARIANT: + case MAP: + case ARRAY: + case NULL: + case UNDEFINED: + default: + throw new IllegalArgumentException("Unexpected value: " + type.dataType()); + } + + return dataType; + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/BigQueryDataTypeToLogicalDataTypeMapping.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/BigQueryDataTypeToLogicalDataTypeMapping.java new file mode 100644 index 00000000000..6be350a7055 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/BigQueryDataTypeToLogicalDataTypeMapping.java @@ -0,0 +1,71 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql; + +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.relational.sql.JdbcPropertiesToLogicalDataTypeMapping; + +public class BigQueryDataTypeToLogicalDataTypeMapping implements JdbcPropertiesToLogicalDataTypeMapping +{ + private static final String INT64 = "INT64"; + private static final String INTEGER = "INTEGER"; + private static final String NUMERIC = "NUMERIC"; + private static final String FLOAT64 = "FLOAT64"; + private static final String FLOAT = "FLOAT"; + private static final String STRING = "STRING"; + private static final String BYTES = "BYTES"; + private static final String BOOL = "BOOL"; + private static final String BOOLEAN = "BOOLEAN"; + private static final String DATE = "DATE"; + private static final String TIME = "TIME"; + private static final String DATETIME = "DATETIME"; + private static final String TIMESTAMP = "TIMESTAMP"; + private static final String JSON = "JSON"; + + public FieldType getDataType(String typeName, String dataType, Integer columnSize, Integer decimalDigits) + { + switch (typeName) + { + case INT64: + case INTEGER: + return FieldType.builder().dataType(DataType.INTEGER).build(); + case NUMERIC: + return FieldType.builder().dataType(DataType.NUMERIC).length(columnSize).scale(decimalDigits).build(); + case FLOAT64: + case FLOAT: + return FieldType.builder().dataType(DataType.FLOAT).build(); + case STRING: + return FieldType.builder().dataType(DataType.STRING).length(columnSize).build(); + case BYTES: + return FieldType.builder().dataType(DataType.BYTES).length(columnSize).build(); + case BOOL: + case BOOLEAN: + return FieldType.builder().dataType(DataType.BOOLEAN).build(); + case DATE: + return FieldType.builder().dataType(DataType.DATE).build(); + case TIME: + return FieldType.builder().dataType(DataType.TIME).build(); + case DATETIME: + return FieldType.builder().dataType(DataType.DATETIME).build(); + case TIMESTAMP: + return FieldType.builder().dataType(DataType.TIMESTAMP).build(); + case JSON: + return FieldType.builder().dataType(DataType.JSON).build(); + default: + throw new IllegalArgumentException("Unexpected values: JDBC TYPE_NAME " + typeName + ", JDBC DATA_TYPE: " + dataType); + } + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/AlterVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/AlterVisitor.java new file mode 100644 index 00000000000..88f08d6e05d --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/AlterVisitor.java @@ -0,0 +1,43 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.operations.Alter; +import org.finos.legend.engine.persistence.components.optimizer.Optimizer; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schemaops.statements.AlterTable; +import org.finos.legend.engine.persistence.components.relational.sqldom.common.AlterOperation; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.Arrays; + +public class AlterVisitor implements LogicalPlanVisitor +{ + + @Override + public VisitorResult visit(PhysicalPlanNode prev, Alter current, VisitorContext context) + { + AlterTable alterTable = new AlterTable(AlterOperation.valueOf(current.operation().name())); + for (Optimizer optimizer : context.optimizers()) + { + alterTable = (AlterTable) optimizer.optimize(alterTable); + } + + prev.push(alterTable); + + return new VisitorResult(alterTable, Arrays.asList(current.dataset(), current.columnDetails())); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchEndTimestampVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchEndTimestampVisitor.java new file mode 100644 index 00000000000..454d2bb4a57 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchEndTimestampVisitor.java @@ -0,0 +1,44 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.values.BatchEndTimestamp; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.sqldom.common.FunctionName; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.values.Function; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.Optional; + +public class BatchEndTimestampVisitor implements LogicalPlanVisitor +{ + + @Override + public VisitorResult visit(PhysicalPlanNode prev, BatchEndTimestamp current, VisitorContext context) + { + + Optional batchEndTimestampPattern = context.batchEndTimestampPattern(); + if (batchEndTimestampPattern.isPresent()) + { + prev.push(new org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.values.StringValue(batchEndTimestampPattern.get(), context.quoteIdentifier())); + } + else + { + prev.push(new Function(FunctionName.CURRENT_DATETIME, null, context.quoteIdentifier())); + } + return new VisitorResult(); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchStartTimestampVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchStartTimestampVisitor.java new file mode 100644 index 00000000000..afc2cb9b2f1 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchStartTimestampVisitor.java @@ -0,0 +1,43 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.values.BatchStartTimestamp; +import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionImpl; +import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionName; +import org.finos.legend.engine.persistence.components.logicalplan.values.StringValue; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.FunctionVisitor; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.Optional; + +public class BatchStartTimestampVisitor implements LogicalPlanVisitor +{ + + private static final String DATE_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"; + + @Override + public VisitorResult visit(PhysicalPlanNode prev, BatchStartTimestamp current, VisitorContext context) + { + Optional batchStartTimestampPattern = context.batchStartTimestampPattern(); + StringValue dateTimeFormat = StringValue.of(DATE_TIME_FORMAT); + StringValue datetimeValue; + datetimeValue = StringValue.of(batchStartTimestampPattern.orElse(context.batchStartTimestamp())); + FunctionImpl parseDateTime = FunctionImpl.builder().functionName(FunctionName.PARSE_DATETIME).addValue(dateTimeFormat, datetimeValue).build(); + return new FunctionVisitor().visit(prev, parseDateTime, context); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/ClusterKeyVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/ClusterKeyVisitor.java new file mode 100644 index 00000000000..f204f5cd98f --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/ClusterKeyVisitor.java @@ -0,0 +1,40 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanNode; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.ClusterKey; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.table.ClusteringKeyConstraint; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.ArrayList; +import java.util.List; + +public class ClusterKeyVisitor implements LogicalPlanVisitor +{ + @Override + public LogicalPlanVisitor.VisitorResult visit(PhysicalPlanNode prev, ClusterKey current, VisitorContext context) + { + ClusteringKeyConstraint clusterKey = new ClusteringKeyConstraint(); + prev.push(clusterKey); + + List logicalPlanNodes = new ArrayList<>(); + logicalPlanNodes.add(current.key()); + + return new LogicalPlanVisitor.VisitorResult(clusterKey, logicalPlanNodes); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/DeleteVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/DeleteVisitor.java new file mode 100644 index 00000000000..d0cdbfb593d --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/DeleteVisitor.java @@ -0,0 +1,52 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanNode; +import org.finos.legend.engine.persistence.components.logicalplan.conditions.Condition; +import org.finos.legend.engine.persistence.components.logicalplan.conditions.Equals; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Delete; +import org.finos.legend.engine.persistence.components.logicalplan.values.ObjectValue; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.statements.DeleteStatement; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +public class DeleteVisitor implements LogicalPlanVisitor +{ + + /* + DELETE always needs A WHERE CLAUSE IN BigQuery + If the condition is not provided, default condition used: 1 = 1 + */ + + @Override + public VisitorResult visit(PhysicalPlanNode prev, Delete current, VisitorContext context) + { + Condition condition = current.condition().orElseGet(() -> Equals.of(ObjectValue.of(1), ObjectValue.of(1))); + DeleteStatement deleteStatement = new DeleteStatement(); + prev.push(deleteStatement); + + List logicalPlanNodeList = new ArrayList<>(); + logicalPlanNodeList.add(current.dataset()); + logicalPlanNodeList.add(condition); + + return new VisitorResult(deleteStatement, logicalPlanNodeList); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/FieldVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/FieldVisitor.java new file mode 100644 index 00000000000..ccb972dea69 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/FieldVisitor.java @@ -0,0 +1,26 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.BigQueryDataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; + +public class FieldVisitor extends org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.FieldVisitor +{ + public DataTypeMapping getDataTypeMapping() + { + return new BigQueryDataTypeMapping(); + } +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/PartitionKeyVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/PartitionKeyVisitor.java new file mode 100644 index 00000000000..185e4d790ad --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/PartitionKeyVisitor.java @@ -0,0 +1,40 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanNode; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.PartitionKey; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.table.PartitionKeyConstraint; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.ArrayList; +import java.util.List; + +public class PartitionKeyVisitor implements LogicalPlanVisitor +{ + @Override + public VisitorResult visit(PhysicalPlanNode prev, PartitionKey current, VisitorContext context) + { + PartitionKeyConstraint partitionKeyConstraint = new PartitionKeyConstraint(); + prev.push(partitionKeyConstraint); + + List logicalPlanNodes = new ArrayList<>(); + logicalPlanNodes.add(current.key()); + + return new VisitorResult(partitionKeyConstraint, logicalPlanNodes); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/SQLCreateVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/SQLCreateVisitor.java new file mode 100644 index 00000000000..28e359890ac --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/SQLCreateVisitor.java @@ -0,0 +1,60 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanNode; +import org.finos.legend.engine.persistence.components.logicalplan.modifiers.IfNotExistsTableModifier; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Create; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schemaops.statements.CreateTable; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.ArrayList; +import java.util.List; + +public class SQLCreateVisitor implements LogicalPlanVisitor +{ + + @Override + public VisitorResult visit(PhysicalPlanNode prev, Create current, VisitorContext context) + { + CreateTable createTable = new CreateTable(); + prev.push(createTable); + + List logicalPlanNodes = new ArrayList<>(); + logicalPlanNodes.add(current.dataset().datasetReference()); + logicalPlanNodes.add(current.dataset().schema()); + + if (current.ifNotExists()) + { + logicalPlanNodes.add(IfNotExistsTableModifier.INSTANCE); + } + + // Add Partition Keys + if (current.dataset().schema().partitionKeys() != null && !current.dataset().schema().partitionKeys().isEmpty()) + { + logicalPlanNodes.addAll(current.dataset().schema().partitionKeys()); + } + + // Add Clustering Keys + if (current.dataset().schema().clusterKeys() != null && !current.dataset().schema().clusterKeys().isEmpty()) + { + logicalPlanNodes.addAll(current.dataset().schema().clusterKeys()); + } + + return new VisitorResult(createTable, logicalPlanNodes); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/SchemaDefinitionVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/SchemaDefinitionVisitor.java new file mode 100644 index 00000000000..4fcbda658e4 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/SchemaDefinitionVisitor.java @@ -0,0 +1,84 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; +import org.finos.legend.engine.persistence.components.optimizer.Optimizer; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.bigquery.sql.BigQueryDataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.constraints.columns.PKColumnConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.ColumnConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.NotNullColumnConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.table.PrimaryKeyTableConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.table.TableConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.DataType; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.Column; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class SchemaDefinitionVisitor implements LogicalPlanVisitor +{ + /* + IN BigQuery World + Project => Database + Dataset => Schema + Table Name => Table + */ + + @Override + public VisitorResult visit(PhysicalPlanNode prev, SchemaDefinition current, VisitorContext context) + { + + List pkFields = current.fields().stream().filter(Field::primaryKey).collect(Collectors.toList()); + int pkNum = pkFields.size(); + + for (Field f : current.fields()) + { + DataType dataType = new BigQueryDataTypeMapping().getDataType(f.type()); + List columnConstraints = new ArrayList<>(); + if (!f.nullable()) + { + columnConstraints.add(new NotNullColumnConstraint()); + } + if (f.primaryKey() && pkNum <= 1) + { + columnConstraints.add(new PKColumnConstraint()); + } + Column column = new Column(f.name(), dataType, columnConstraints, context.quoteIdentifier()); + for (Optimizer optimizer : context.optimizers()) + { + column = (Column) optimizer.optimize(column); + } + prev.push(column); + } + + if (pkNum > 1) + { + TableConstraint constraint = new PrimaryKeyTableConstraint(pkFields.stream().map(Field::name).collect(Collectors.toList()), context.quoteIdentifier(), true); + for (Optimizer optimizer : context.optimizers()) + { + constraint = (TableConstraint) optimizer.optimize(constraint); + } + prev.push(constraint); + } + + return new VisitorResult(null); + } +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/TruncateVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/TruncateVisitor.java new file mode 100644 index 00000000000..d1a8b4f56e4 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/TruncateVisitor.java @@ -0,0 +1,47 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; + +import org.finos.legend.engine.persistence.components.logicalplan.operations.Truncate; +import org.finos.legend.engine.persistence.components.optimizer.Optimizer; +import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.statements.TruncateTable; +import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; +import org.finos.legend.engine.persistence.components.transformer.VisitorContext; + +import java.util.Collections; + +public class TruncateVisitor implements LogicalPlanVisitor +{ + + @Override + public VisitorResult visit(PhysicalPlanNode prev, Truncate current, VisitorContext context) + { + // TODO For partitioned Tables, if the table requires a partition filter, Truncate will fail. + // This should be the approach in that case: + // 1. UPDATE the table to remove the partition filter requirement + // 2. Truncate table + // 3. UPDATE the table to add the partition filter requirement + + TruncateTable truncateTable = new TruncateTable(); + for (Optimizer optimizer : context.optimizers()) + { + truncateTable = (TruncateTable) optimizer.optimize(truncateTable); + } + prev.push(truncateTable); + + return new VisitorResult(truncateTable, Collections.singletonList(current.dataset())); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/constraints/columns/PKColumnConstraint.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/constraints/columns/PKColumnConstraint.java new file mode 100644 index 00000000000..1419243efd9 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/constraints/columns/PKColumnConstraint.java @@ -0,0 +1,26 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.constraints.columns; + +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.ColumnConstraint; + +public class PKColumnConstraint extends ColumnConstraint +{ + @Override + public void genSql(StringBuilder builder) + { + builder.append("PRIMARY KEY NOT ENFORCED"); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Bool.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Bool.java new file mode 100644 index 00000000000..4a8f2c9ad3a --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Bool.java @@ -0,0 +1,26 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema; + +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.VariableSizeDataType; + +public class Bool extends VariableSizeDataType +{ + + public Bool() + { + super("BOOL"); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Bytes.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Bytes.java new file mode 100644 index 00000000000..575d0ef5724 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Bytes.java @@ -0,0 +1,32 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema; + +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.VariableSizeDataType; + +public class Bytes extends VariableSizeDataType +{ + + public Bytes() + { + super("BYTES"); + } + + public Bytes(int sz) + { + super("BYTES", sz); + } + +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Float64.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Float64.java new file mode 100644 index 00000000000..849fcb485c2 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Float64.java @@ -0,0 +1,27 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema; + +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.VariableSizeDataType; + +public class Float64 extends VariableSizeDataType +{ + + public Float64() + { + super("FLOAT64"); + } + +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Json.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Json.java new file mode 100644 index 00000000000..46c67f3a3cf --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/Json.java @@ -0,0 +1,26 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema; + +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.VariableSizeDataType; + +public class Json extends VariableSizeDataType +{ + + public Json() + { + super("JSON"); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/String.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/String.java new file mode 100644 index 00000000000..6fa7bdd6d3d --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schema/String.java @@ -0,0 +1,32 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schema; + +import org.finos.legend.engine.persistence.components.relational.sqldom.schema.VariableSizeDataType; + +public class String extends VariableSizeDataType +{ + + public String() + { + super("STRING"); + } + + public String(int sz) + { + super("STRING", sz); + } + +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/AlterTable.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/AlterTable.java new file mode 100644 index 00000000000..70c9ead4f8f --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/AlterTable.java @@ -0,0 +1,132 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schemaops.statements; + +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlDomException; +import org.finos.legend.engine.persistence.components.relational.sqldom.common.AlterOperation; +import org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.column.NotNullColumnConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.Column; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.expresssions.table.Table; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.statements.DDLStatement; + +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.COLUMN; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.DROP; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.WHITE_SPACE; + +public class AlterTable implements DDLStatement +{ + private final AlterOperation operation; + private Table table; + private Column columnToAlter; + + public AlterTable(AlterOperation operation) + { + this.operation = operation; + } + + public Table getTable() + { + return table; + } + + /* + Add column: + ALTER TABLE [TABLE_NAME] ADD COLUMN [NEW_COLUMN_NAME] [COLUMN_DATATYPE] + Drop Not Null: + ALTER TABLE [TABLE_NAME] ALTER COLUMN [COLUMN_NAME] DROP NOT NULL + Change Data type: + ALTER TABLE [TABLE_NAME] ALTER COLUMN [COLUMN_NAME] SET DATA TYPE column_schema + https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#alter_column_set_data_type_statement + */ + + @Override + public void genSql(StringBuilder builder) throws SqlDomException + { + validate(); + builder.append(Clause.ALTER.get()); + + builder.append(WHITE_SPACE + Clause.TABLE.get()); + + // Table name + builder.append(WHITE_SPACE); + table.genSqlWithoutAlias(builder); + + // Operation + builder.append(WHITE_SPACE); + if (operation.getParent() == null) + { + builder.append(operation.name()); + } + else + { + builder.append(operation.getParent().name()); + } + builder.append(WHITE_SPACE); + builder.append(COLUMN); + builder.append(WHITE_SPACE); + + switch (operation) + { + case ADD: + columnToAlter.genSql(builder); + break; + case CHANGE_DATATYPE: + columnToAlter.genSqlWithNameOnly(builder); + builder.append(WHITE_SPACE); + builder.append(Clause.SET.get()); + builder.append(WHITE_SPACE); + builder.append(Clause.DATA_TYPE.get()); + builder.append(WHITE_SPACE); + columnToAlter.genSqlWithTypeOnly(builder); + break; + case NULLABLE_COLUMN: + columnToAlter.genSqlWithNameOnly(builder); + builder.append(WHITE_SPACE); + builder.append(DROP); + builder.append(WHITE_SPACE); + NotNullColumnConstraint notNullColumnConstraint = new NotNullColumnConstraint(); + notNullColumnConstraint.genSql(builder); + break; + default: + throw new SqlDomException("Alter operation " + operation.name() + " not supported"); + } + } + + @Override + public void push(Object node) + { + if (node instanceof Table) + { + table = (Table) node; + } + else if (node instanceof Column) + { + columnToAlter = (Column) node; + } + } + + void validate() throws SqlDomException + { + if (table == null) + { + throw new SqlDomException("Table is mandatory for Alter Table Command"); + } + if (columnToAlter == null) + { + throw new SqlDomException("Columns details is mandatory for Alter Table Command"); + } + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/CreateTable.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/CreateTable.java new file mode 100644 index 00000000000..6f687dbe82f --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/CreateTable.java @@ -0,0 +1,173 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sqldom.schemaops.statements; + +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlDomException; +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; +import org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.table.ClusteringKeyConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.table.PartitionKeyConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.constraints.table.TableConstraint; +import org.finos.legend.engine.persistence.components.relational.sqldom.modifiers.TableModifier; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.Column; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.expresssions.table.Table; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.statements.DDLStatement; +import org.finos.legend.engine.persistence.components.relational.sqldom.tabletypes.TableType; + +import java.util.ArrayList; +import java.util.List; + +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.CLOSING_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.COMMA; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.EMPTY; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.OPEN_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.WHITE_SPACE; + +public class CreateTable implements DDLStatement +{ + private Table table; + private final List modifiers; // dataset + private final List columns; // schema + private final List tableConstraints; // table level + private final List types; // dataset + private final List clusterKeys; + private final List partitionKeys; + + public CreateTable() + { + this.modifiers = new ArrayList<>(); + this.columns = new ArrayList<>(); + this.tableConstraints = new ArrayList<>(); + this.types = new ArrayList<>(); + this.clusterKeys = new ArrayList<>(); + this.partitionKeys = new ArrayList<>(); + } + + public CreateTable(Table table, List modifiers, List columns, List tableConstraints, List types) + { + this.table = table; + this.modifiers = modifiers; + this.columns = columns; + this.tableConstraints = tableConstraints; + this.types = types; + this.clusterKeys = new ArrayList<>(); + this.partitionKeys = new ArrayList<>(); + } + + public CreateTable(List types) + { + this.modifiers = new ArrayList<>(); + this.columns = new ArrayList<>(); + this.tableConstraints = new ArrayList<>(); + this.types = types; + this.clusterKeys = new ArrayList<>(); + this.partitionKeys = new ArrayList<>(); + } + + /* + CREATE GENERIC PLAN: + CREATE [TABLE TYPE] TABLE [IF NOT EXISTS] {FULLY_QUALIFIED_TABLE_NAME}( {COLUMNS}{CONSTRAINTS} ) + PARTITION BY <> + CLUSTER BY <> + */ + @Override + public void genSql(StringBuilder builder) throws SqlDomException + { + validate(); + builder.append(Clause.CREATE.get()); + + // Table Type + SqlGen.genSqlList(builder, types, WHITE_SPACE, WHITE_SPACE); + + builder.append(WHITE_SPACE + Clause.TABLE.get()); + + // Modifiers + SqlGen.genSqlList(builder, modifiers, WHITE_SPACE, WHITE_SPACE); + + // Table name + builder.append(WHITE_SPACE); + table.genSqlWithoutAlias(builder); + + // Columns + table constraints + builder.append(OPEN_PARENTHESIS); + SqlGen.genSqlList(builder, columns, EMPTY, COMMA); + SqlGen.genSqlList(builder, tableConstraints, COMMA, COMMA); + builder.append(CLOSING_PARENTHESIS); + + // Partition Key Expression + if (!partitionKeys.isEmpty()) + { + builder.append(WHITE_SPACE + Clause.PARTITION_BY.get() + WHITE_SPACE); + SqlGen.genSqlList(builder, partitionKeys, EMPTY, COMMA); + } + + // Clustering keys + if (!clusterKeys.isEmpty()) + { + builder.append(WHITE_SPACE + Clause.CLUSTER_BY.get() + WHITE_SPACE); + SqlGen.genSqlList(builder, clusterKeys, EMPTY, COMMA); + } + } + + + @Override + public void push(Object node) + { + if (node instanceof Table) + { + table = (Table) node; + } + else if (node instanceof TableType) + { + types.add((TableType) node); + } + else if (node instanceof TableModifier) + { + modifiers.add((TableModifier) node); + } + else if (node instanceof Column) + { + columns.add((Column) node); + } + else if (node instanceof TableConstraint) + { + tableConstraints.add((TableConstraint) node); + } + else if (node instanceof ClusteringKeyConstraint) + { + clusterKeys.add((ClusteringKeyConstraint) node); + } + else if (node instanceof PartitionKeyConstraint) + { + partitionKeys.add((PartitionKeyConstraint) node); + } + } + + void validate() throws SqlDomException + { + if (table == null) + { + throw new SqlDomException("Table is mandatory for Create Table Command"); + } + if (columns == null || columns.isEmpty()) + { + throw new SqlDomException("Columns list is mandatory for Create Table Command"); + } + if (!partitionKeys.isEmpty() && partitionKeys.size() != 1) + { + throw new SqlDomException("Only one Partition Key expression is allowed for BigQuery Create Table Command"); + } + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/BaseTestUtils.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/BaseTestUtils.java new file mode 100644 index 00000000000..d604902d3a5 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/BaseTestUtils.java @@ -0,0 +1,179 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components; + +import org.finos.legend.engine.persistence.components.logicalplan.datasets.ClusterKey; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.PartitionKey; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.values.FieldValue; +import org.finos.legend.engine.persistence.components.logicalplan.values.ObjectValue; + +import java.util.Optional; + +public class BaseTestUtils +{ + public static Field colInt = Field.builder().name("col_int").type(FieldType.of(DataType.INT, Optional.empty(), Optional.empty())).primaryKey(true).build(); + public static Field colInteger = Field.builder().name("col_integer").type(FieldType.of(DataType.INTEGER, Optional.empty(), Optional.empty())).nullable(false).build(); + public static Field colBigint = Field.builder().name("col_bigint").type(FieldType.of(DataType.BIGINT, Optional.empty(), Optional.empty())).build(); + public static Field colTinyint = Field.builder().name("col_tinyint").type(FieldType.of(DataType.TINYINT, Optional.empty(), Optional.empty())).build(); + public static Field colSmallint = Field.builder().name("col_smallint").type(FieldType.of(DataType.SMALLINT, Optional.empty(), Optional.empty())).build(); + public static Field colInt64 = Field.builder().name("col_int64").type(FieldType.of(DataType.INT64, Optional.empty(), Optional.empty())).build(); + public static Field colNumber = Field.builder().name("col_number").type(FieldType.of(DataType.NUMBER, Optional.empty(), Optional.empty())).build(); + public static Field colNumeric = Field.builder().name("col_numeric").type(FieldType.of(DataType.NUMERIC, Optional.empty(), Optional.empty())).build(); + public static Field colNumericWithPrecision = Field.builder().name("col_numeric_with_precision").type(FieldType.of(DataType.NUMERIC, Optional.of(29), Optional.empty())).build(); + public static Field colNumericWithScale = Field.builder().name("col_numeric_with_scale").type(FieldType.of(DataType.NUMERIC, Optional.of(33), Optional.of(4))).build(); + public static Field colDecimal = Field.builder().name("col_decimal").type(FieldType.of(DataType.DECIMAL, Optional.empty(), Optional.empty())).build(); + public static Field colReal = Field.builder().name("col_real").type(FieldType.of(DataType.REAL, Optional.empty(), Optional.empty())).build(); + public static Field colFloat = Field.builder().name("col_float").type(FieldType.of(DataType.FLOAT, Optional.empty(), Optional.empty())).build(); + public static Field colDouble = Field.builder().name("col_double").type(FieldType.of(DataType.DOUBLE, Optional.empty(), Optional.empty())).build(); + public static Field colFloat64 = Field.builder().name("col_float64").type(FieldType.of(DataType.FLOAT64, Optional.empty(), Optional.empty())).build(); + public static Field colChar = Field.builder().name("col_char").type(FieldType.of(DataType.CHAR, Optional.empty(), Optional.empty())).build(); + public static Field colCharacter = Field.builder().name("col_character").type(FieldType.of(DataType.CHARACTER, Optional.empty(), Optional.empty())).build(); + public static Field colVarchar = Field.builder().name("col_varchar").type(FieldType.of(DataType.VARCHAR, Optional.empty(), Optional.empty())).build(); + public static Field colLongVarchar = Field.builder().name("col_longvarchar").type(FieldType.of(DataType.LONGVARCHAR, Optional.empty(), Optional.empty())).build(); + public static Field colLongtext = Field.builder().name("col_longtext").type(FieldType.of(DataType.LONGTEXT, Optional.empty(), Optional.empty())).build(); + public static Field colText = Field.builder().name("col_text").type(FieldType.of(DataType.TEXT, Optional.empty(), Optional.empty())).build(); + public static Field colString = Field.builder().name("col_string").type(FieldType.of(DataType.STRING, Optional.empty(), Optional.empty())).build(); + public static Field colStringWithLength = Field.builder().name("col_string_with_length").type(FieldType.of(DataType.STRING, Optional.of(16), Optional.empty())).build(); + public static Field colBinary = Field.builder().name("col_binary").type(FieldType.of(DataType.BINARY, Optional.empty(), Optional.empty())).build(); + public static Field colVarBinary = Field.builder().name("col_varbinary").type(FieldType.of(DataType.VARBINARY, Optional.empty(), Optional.empty())).build(); + public static Field colLongVarBinary = Field.builder().name("col_longvarbinary").type(FieldType.of(DataType.LONGVARBINARY, Optional.empty(), Optional.empty())).build(); + public static Field colBytes = Field.builder().name("col_bytes").type(FieldType.of(DataType.BYTES, Optional.empty(), Optional.empty())).build(); + public static Field colBytesWithLength = Field.builder().name("col_bytes_with_length").type(FieldType.of(DataType.BYTES, 10, null)).build(); + public static Field colDate = Field.builder().name("col_date").type(FieldType.of(DataType.DATE, Optional.empty(), Optional.empty())).primaryKey(true).build(); + public static Field colTime = Field.builder().name("col_time").type(FieldType.of(DataType.TIME, Optional.empty(), Optional.empty())).build(); + public static Field colDatetime = Field.builder().name("col_datetime").type(FieldType.of(DataType.DATETIME, Optional.empty(), Optional.empty())).build(); + public static Field colTimestamp = Field.builder().name("col_timestamp").type(FieldType.of(DataType.TIMESTAMP, Optional.empty(), Optional.empty())).build(); + public static Field colBoolean = Field.builder().name("col_boolean").type(FieldType.of(DataType.BOOLEAN, Optional.empty(), Optional.empty())).build(); + public static Field colBool = Field.builder().name("col_bool").type(FieldType.of(DataType.BOOL, Optional.empty(), Optional.empty())).build(); + public static Field colJson = Field.builder().name("col_json").type(FieldType.of(DataType.JSON, Optional.empty(), Optional.empty())).build(); + + public static ClusterKey clusterKey1 = ClusterKey.builder().key(FieldValue.builder().fieldName("col_timestamp").build()).build(); + public static ClusterKey clusterKey2 = ClusterKey.builder().key(FieldValue.builder().fieldName("col_int").build()).build(); + + public static PartitionKey partitionKey1 = PartitionKey.of(FieldValue.builder().fieldName("col_date").build()); + + public static PartitionKey partitionKey2 = PartitionKey.of(ObjectValue.of("_PARTITIONDATE")); + + public static SchemaDefinition schemaWithAllColumns = SchemaDefinition.builder() + .addFields(colInt) + .addFields(colInteger) + .addFields(colBigint) + .addFields(colTinyint) + .addFields(colSmallint) + .addFields(colInt64) + .addFields(colNumber) + .addFields(colNumeric) + .addFields(colNumericWithPrecision) + .addFields(colNumericWithScale) + .addFields(colDecimal) + .addFields(colReal) + .addFields(colFloat) + .addFields(colDouble) + .addFields(colFloat64) + .addFields(colChar) + .addFields(colCharacter) + .addFields(colVarchar) + .addFields(colLongVarchar) + .addFields(colLongtext) + .addFields(colText) + .addFields(colString) + .addFields(colStringWithLength) + .addFields(colBinary) + .addFields(colVarBinary) + .addFields(colLongVarBinary) + .addFields(colBytes) + .addFields(colBytesWithLength) + .addFields(colDate) + .addFields(colTime) + .addFields(colDatetime) + .addFields(colTimestamp) + .addFields(colBoolean) + .addFields(colBool) + .addFields(colJson) + .build(); + + + public static SchemaDefinition schemaWithAllColumnsFromDb = SchemaDefinition.builder() + .addFields(colInt.withType(colInt.type().withDataType(DataType.INTEGER))) + .addFields(colInteger) + .addFields(colBigint.withType(colBigint.type().withDataType(DataType.INTEGER))) + .addFields(colTinyint.withType(colTinyint.type().withDataType(DataType.INTEGER))) + .addFields(colSmallint.withType(colSmallint.type().withDataType(DataType.INTEGER))) + .addFields(colInt64.withType(colInt64.type().withDataType(DataType.INTEGER))) + .addFields(colNumber.withType(colNumber.type().withDataType(DataType.NUMERIC))) + .addFields(colNumeric) + .addFields(colNumericWithPrecision) + .addFields(colNumericWithScale) + .addFields(colDecimal.withType(colDecimal.type().withDataType(DataType.NUMERIC))) + .addFields(colReal.withType(colReal.type().withDataType(DataType.FLOAT))) + .addFields(colFloat) + .addFields(colDouble.withType(colDouble.type().withDataType(DataType.FLOAT))) + .addFields(colFloat64.withType(colFloat64.type().withDataType(DataType.FLOAT))) + .addFields(colChar.withType(colChar.type().withDataType(DataType.STRING))) + .addFields(colCharacter.withType(colCharacter.type().withDataType(DataType.STRING))) + .addFields(colVarchar.withType(colVarchar.type().withDataType(DataType.STRING))) + .addFields(colLongVarchar.withType(colLongVarchar.type().withDataType(DataType.STRING))) + .addFields(colLongtext.withType(colLongtext.type().withDataType(DataType.STRING))) + .addFields(colText.withType(colText.type().withDataType(DataType.STRING))) + .addFields(colString) + .addFields(colStringWithLength) + .addFields(colBinary.withType(colBinary.type().withDataType(DataType.BYTES))) + .addFields(colVarBinary.withType(colVarBinary.type().withDataType(DataType.BYTES))) + .addFields(colLongVarBinary.withType(colLongVarBinary.type().withDataType(DataType.BYTES))) + .addFields(colBytes) + .addFields(colBytesWithLength) + .addFields(colDate) + .addFields(colTime) + .addFields(colDatetime) + .addFields(colTimestamp) + .addFields(colBoolean) + .addFields(colBool.withType(colBool.type().withDataType(DataType.BOOLEAN))) + .addFields(colJson) + .build(); + + public static SchemaDefinition schemaWithClusteringKey = SchemaDefinition.builder() + .addFields(colInt) + .addFields(colInteger) + .addFields(colString) + .addFields(colTimestamp) + .addFields(colDouble) + .addClusterKeys(clusterKey1, clusterKey2) + .build(); + + public static SchemaDefinition schemaWithClusteringAndPartitionKey = SchemaDefinition.builder() + .addFields(colInt) + .addFields(colDate) + .addFields(colInteger) + .addFields(colString) + .addFields(colTimestamp) + .addFields(colDouble) + .addClusterKeys(clusterKey1, clusterKey2) + .addPartitionKeys(partitionKey1) + .build(); + + public static SchemaDefinition schemaWithPartitionKey = SchemaDefinition.builder() + .addFields(colInt) + .addFields(colDate) + .addFields(colInteger) + .addFields(colString) + .addFields(colTimestamp) + .addFields(colDouble) + .addPartitionKeys(partitionKey2) + .build(); +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/AppendOnlyExecutorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/AppendOnlyExecutorTest.java new file mode 100644 index 00000000000..c1fec8890b9 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/AppendOnlyExecutorTest.java @@ -0,0 +1,86 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.common.StatisticName; +import org.finos.legend.engine.persistence.components.ingestmode.AppendOnly; +import org.finos.legend.engine.persistence.components.ingestmode.audit.DateTimeAuditing; +import org.finos.legend.engine.persistence.components.ingestmode.deduplication.FilterDuplicates; +import org.finos.legend.engine.persistence.components.relational.api.IngestorResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.finos.legend.engine.persistence.components.common.StatisticName.INCOMING_RECORD_COUNT; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_INSERTED; + +@Disabled +public class AppendOnlyExecutorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + AppendOnly ingestMode = AppendOnly.builder() + .digestField("digest") + .deduplicationStrategy(FilterDuplicates.builder().build()) + .auditing(DateTimeAuditing.builder().dateTimeField("audit_ts").build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + IngestorResult result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/append/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts", "audit_ts"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + Map map = result.statisticByName(); + + long incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + long rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + Assertions.assertEquals(3, incomingRecords); + Assertions.assertEquals(3, rowsInserted); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc, insert_ts"); + expectedPath = "src/test/resources/expected/append/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + Assertions.assertEquals(3, incomingRecords); + Assertions.assertEquals(2, rowsInserted); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/AppendOnlyGeneratorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/AppendOnlyGeneratorTest.java new file mode 100644 index 00000000000..6ffa34b0909 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/AppendOnlyGeneratorTest.java @@ -0,0 +1,69 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.AppendOnly; +import org.finos.legend.engine.persistence.components.ingestmode.audit.DateTimeAuditing; +import org.finos.legend.engine.persistence.components.ingestmode.deduplication.FilterDuplicates; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@Disabled +public class AppendOnlyGeneratorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + AppendOnly ingestMode = AppendOnly.builder() + .digestField("digest") + .deduplicationStrategy(FilterDuplicates.builder().build()) + .auditing(DateTimeAuditing.builder().dateTimeField("audit_ts").build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/append/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts", "audit_ts"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc, insert_ts"); + expectedPath = "src/test/resources/expected/append/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BigQueryEndToEndTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BigQueryEndToEndTest.java new file mode 100644 index 00000000000..2335c194af5 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BigQueryEndToEndTest.java @@ -0,0 +1,422 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import com.google.auth.oauth2.GoogleCredentials; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.InsertAllRequest; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.TableId; +import com.opencsv.CSVReader; +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.Datasets; +import org.finos.legend.engine.persistence.components.common.StatisticName; +import org.finos.legend.engine.persistence.components.ingestmode.IngestMode; +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanFactory; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DerivedDataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; +import org.finos.legend.engine.persistence.components.planner.PlannerOptions; +import org.finos.legend.engine.persistence.components.relational.CaseConversion; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.api.IngestorResult; +import org.finos.legend.engine.persistence.components.relational.api.RelationalConnection; +import org.finos.legend.engine.persistence.components.relational.api.RelationalGenerator; +import org.finos.legend.engine.persistence.components.relational.api.RelationalIngestor; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.relational.bigquery.executor.BigQueryConnection; +import org.finos.legend.engine.persistence.components.relational.bigquery.executor.BigQueryHelper; +import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; +import org.finos.legend.engine.persistence.components.util.MetadataDataset; +import org.finos.legend.engine.persistence.components.util.SchemaEvolutionCapability; +import org.junit.jupiter.api.Assertions; + +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +public class BigQueryEndToEndTest +{ + protected static String digestName = "digest"; + protected Field id = Field.builder().name("id").type(FieldType.of(DataType.INT, Optional.empty(), Optional.empty())).primaryKey(true).build(); + protected Field name = Field.builder().name("name").type(FieldType.of(DataType.VARCHAR, Optional.empty(), Optional.empty())).primaryKey(true).build(); + protected Field amount = Field.builder().name("amount").type(FieldType.of(DataType.INTEGER, Optional.empty(), Optional.empty())).build(); + protected Field bizDate = Field.builder().name("biz_date").type(FieldType.of(DataType.DATE, Optional.empty(), Optional.empty())).build(); + protected Field digest = Field.builder().name(digestName).type(FieldType.of(DataType.STRING, Optional.empty(), Optional.empty())).build(); + protected Field insertTimestamp = Field.builder().name("insert_ts").type(FieldType.of(DataType.TIMESTAMP, Optional.empty(), Optional.empty())).build(); + protected final ZonedDateTime fixedZonedDateTime_2000_01_01 = ZonedDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); + protected final ZonedDateTime fixedZonedDateTime_2000_01_02 = ZonedDateTime.of(2000, 1, 2, 0, 0, 0, 0, ZoneOffset.UTC); + protected final ZonedDateTime fixedZonedDateTime_2000_01_03 = ZonedDateTime.of(2000, 1, 2, 0, 0, 0, 0, ZoneOffset.UTC); + protected final ZonedDateTime fixedZonedDateTime_2000_01_04 = ZonedDateTime.of(2000, 1, 2, 0, 0, 0, 0, ZoneOffset.UTC); + protected final ZonedDateTime fixedZonedDateTime_2000_01_05 = ZonedDateTime.of(2000, 1, 2, 0, 0, 0, 0, ZoneOffset.UTC); + protected final ZonedDateTime fixedZonedDateTime_2000_01_06 = ZonedDateTime.of(2000, 1, 2, 0, 0, 0, 0, ZoneOffset.UTC); + + protected final Clock fixedClock_2000_01_01 = Clock.fixed(fixedZonedDateTime_2000_01_01.toInstant(), ZoneOffset.UTC); + protected final Clock fixedClock_2000_01_02 = Clock.fixed(fixedZonedDateTime_2000_01_02.toInstant(), ZoneOffset.UTC); + protected final Clock fixedClock_2000_01_03 = Clock.fixed(fixedZonedDateTime_2000_01_03.toInstant(), ZoneOffset.UTC); + protected final Clock fixedClock_2000_01_04 = Clock.fixed(fixedZonedDateTime_2000_01_04.toInstant(), ZoneOffset.UTC); + protected final Clock fixedClock_2000_01_05 = Clock.fixed(fixedZonedDateTime_2000_01_05.toInstant(), ZoneOffset.UTC); + protected final Clock fixedClock_2000_01_06 = Clock.fixed(fixedZonedDateTime_2000_01_06.toInstant(), ZoneOffset.UTC); + + // replace these values with actual project id and service account credential path + // for creating a service account, refer https://cloud.google.com/iam/docs/service-accounts-create#creating_a_service_account + // and for getting the service account credential, refer https://cloud.google.com/iam/docs/keys-create-delete + protected final String projectId = "dummy-value"; + protected final String credentialPath = "dummy-value"; + + protected SchemaDefinition stagingSchema = SchemaDefinition.builder() + .addFields(id) // PK + .addFields(name) // PK + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(insertTimestamp) + .build(); + + protected DatasetDefinition mainDataset = DatasetDefinition.builder() + .group("demo").name("main").alias("sink") + .schema(SchemaDefinition.builder().build()) + .build(); + + MetadataDataset metadataDataset = MetadataDataset.builder().metadataDatasetGroupName("demo").metadataDatasetName("batch_metadata").build(); + + public static String batchIdInName = "batch_id_in"; + public static String batchIdOutName = "batch_id_out"; + public static String indexName = "index"; + public static String dateTimeName = "datetime"; + public static String balanceName = "balance"; + public static String startDateTimeName = "start_datetime"; + public static String endDateTimeName = "end_datetime"; + public static Field index = Field.builder().name(indexName).type(FieldType.of(DataType.INT, Optional.empty(), Optional.empty())).primaryKey(true).fieldAlias(indexName).build(); + public static Field dateTime = Field.builder().name(dateTimeName).type(FieldType.of(DataType.DATETIME, Optional.empty(), Optional.empty())).fieldAlias(dateTimeName).primaryKey(true).build(); + public static Field balance = Field.builder().name(balanceName).type(FieldType.of(DataType.BIGINT, Optional.empty(), Optional.empty())).fieldAlias(balanceName).build(); + protected SchemaDefinition bitempStagingSchema = SchemaDefinition.builder() + .addFields(index) + .addFields(dateTime) + .addFields(balance) + .addFields(digest) + .build(); + + protected IngestorResult ingestViaExecutor(IngestMode ingestMode, SchemaDefinition stagingSchema, DatasetFilter stagingFilter, String path, Clock clock) throws IOException, InterruptedException + { + RelationalIngestor ingestor = RelationalIngestor.builder() + .ingestMode(ingestMode) + .relationalSink(BigQuerySink.get()) + .collectStatistics(true) + .cleanupStagingData(false) + .executionTimestampClock(clock) + .build(); + + DerivedDataset stagingDataset = DerivedDataset.builder() + .group("demo") + .name("staging") + .alias("stage") + .schema(stagingSchema) + .addDatasetFilters(stagingFilter) + .build(); + Datasets datasets = Datasets.builder().mainDataset(mainDataset).stagingDataset(stagingDataset).metadataDataset(metadataDataset).build(); + + // Load csv data + loadData(path, datasets.stagingDataset(), 1); + RelationalConnection connection = BigQueryConnection.of(getBigQueryConnection()); + IngestorResult ingestorResult = ingestor.ingest(connection, datasets); + return ingestorResult; + } + + + protected void ingestViaGenerator(IngestMode ingestMode, SchemaDefinition stagingSchema, DatasetFilter stagingFilter, String path, Clock clock) throws IOException, InterruptedException + { + + RelationalGenerator generator = RelationalGenerator.builder() + .ingestMode(ingestMode) + .relationalSink(BigQuerySink.get()) + .collectStatistics(true) + .cleanupStagingData(false) + .executionTimestampClock(clock) + .build(); + + DerivedDataset stagingDataset = DerivedDataset.builder() + .group("demo") + .name("staging") + .alias("stage") + .schema(stagingSchema) + .addDatasetFilters(stagingFilter) + .build(); + Datasets datasets = Datasets.builder().mainDataset(mainDataset).stagingDataset(stagingDataset).metadataDataset(metadataDataset).build(); + + // Load csv data + loadData(path, datasets.stagingDataset(), 1); + + GeneratorResult operations = generator.generateOperations(datasets); + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + List postActionsSql = operations.postActionsSql(); + + // Perform ingestion + ingest(preActionsSqlList, milestoningSqlList, metadataIngestSql, postActionsSql); + } + + void delete(String dataset, String name) throws IOException, InterruptedException + { + String sql = "DROP TABLE IF EXISTS " + String.format("`%s`.`%s`", dataset, name); + runQueries(Arrays.asList(sql)); + } + + void ingest(List preActionsSqlList, List milestoningSqlList, List metadataIngestSql, List postActionsSql) throws IOException, InterruptedException + { + runQueries(preActionsSqlList); + runQueries(milestoningSqlList); + runQueries(metadataIngestSql); + runQueries(postActionsSql); + } + + void createTable(org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset stagingTable) throws InterruptedException, IOException + { + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get()); + LogicalPlan tableCreationPlan = LogicalPlanFactory.getDatasetCreationPlan(stagingTable, true); + SqlPlan tableCreationPhysicalPlan = transformer.generatePhysicalPlan(tableCreationPlan); + List sqlList = tableCreationPhysicalPlan.getSqlList(); + runQueries(sqlList); + } + + protected BigQuery getBigQueryConnection() throws IOException + { + GoogleCredentials credentials = GoogleCredentials.fromStream(new FileInputStream(credentialPath)); + BigQuery bigquery = BigQueryOptions.newBuilder().setCredentials(credentials).setProjectId(projectId).build().getService(); + return bigquery; + } + + private void runQueries(List sqlList) throws IOException, InterruptedException + { + if (sqlList == null || sqlList.isEmpty()) + { + return; + } + String sqls = String.join(";", sqlList); + System.out.println("Running: " + sqls); + BigQuery bigQuery = getBigQueryConnection(); + bigQuery.query(QueryJobConfiguration.newBuilder(sqls).build()); + } + + protected List> runQuery(String sql) throws IOException + { + BigQuery bigQuery = getBigQueryConnection(); + BigQueryHelper helper = BigQueryHelper.of(bigQuery); + return helper.executeQuery(sql); + } + + public static void assertFileAndTableDataEquals(String[] csvSchema, String csvPath, List> dataFromTable) throws IOException + { + List lines = readCsvData(csvPath); + Assertions.assertEquals(lines.size(), dataFromTable.size()); + + for (int i = 0; i < lines.size(); i++) + { + Map tableRow = dataFromTable.get(i); + String[] expectedLine = lines.get(i); + for (int j = 0; j < csvSchema.length; j++) + { + String expected = expectedLine[j]; + Object value = tableRow.get(csvSchema[j]); + if (value instanceof Instant) + { + Instant instant = (Instant) value; + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.of("UTC")); + value = formatter.format(instant); + } + String tableData = String.valueOf(value); + Assertions.assertEquals(expected, tableData); + } + } + } + + void loadData(String path, Dataset stagingDataset, int attempt) throws IOException, InterruptedException + { + // Create Staging table + createTable(stagingDataset); + + attempt++; + String tableName = stagingDataset.datasetReference().name().get(); + String datasetName = stagingDataset.datasetReference().group().get(); + List schema = stagingDataset.schema().fields().stream().map(field -> field.name()).collect(Collectors.toList()); + BigQuery bigQuery = getBigQueryConnection(); + + TableId tableId = TableId.of(datasetName, tableName); + List rows = new ArrayList<>(); + + List csvLines = readCsvData(path); + for (String[] line : csvLines) + { + int i = 0; + Map map = new HashMap<>(); + for (String field : schema) + { + map.put(field, line[i++]); + } + InsertAllRequest.RowToInsert rowToInsert = InsertAllRequest.RowToInsert.of(map); + rows.add(rowToInsert); + } + + if (!rows.isEmpty()) + { + InsertAllRequest insertAllRequest = InsertAllRequest.newBuilder(tableId).setRows(rows).build(); + try + { + bigQuery.insertAll(insertAllRequest); + } + catch (Exception e) + { + System.out.println("Error occurred: " + e.getMessage()); + if (attempt <= 10) + { + // Retry + Thread.sleep(5000); + loadData(path, stagingDataset, attempt); + } + else + { + throw e; + } + } + } + } + + private static List readCsvData(String path) throws IOException + { + FileReader fileReader = new FileReader(path); + CSVReader csvReader = new CSVReader(fileReader); + List lines = csvReader.readAll(); + return lines; + } + + protected Map createExpectedStatsMap(int incomingRecordCount, int rowsDeleted, int rowsInserted, int rowsUpdated, int rowsTerminated) + { + Map expectedStats = new HashMap<>(); + expectedStats.put(StatisticName.INCOMING_RECORD_COUNT.name(), incomingRecordCount); + expectedStats.put(StatisticName.ROWS_DELETED.name(), rowsDeleted); + expectedStats.put(StatisticName.ROWS_INSERTED.name(), rowsInserted); + expectedStats.put(StatisticName.ROWS_TERMINATED.name(), rowsTerminated); + expectedStats.put(StatisticName.ROWS_UPDATED.name(), rowsUpdated); + return expectedStats; + } + + + public IngestorResult executePlansAndVerifyForCaseConversion(IngestMode ingestMode, PlannerOptions options, Datasets datasets, String[] schema, String expectedDataPath, Map expectedStats) throws Exception + { + return executePlansAndVerifyForCaseConversion(ingestMode, options, datasets, schema, expectedDataPath, expectedStats, Clock.systemUTC()); + } + + public IngestorResult executePlansAndVerifyForCaseConversion(IngestMode ingestMode, PlannerOptions options, Datasets datasets, String[] schema, String expectedDataPath, Map expectedStats, Clock executionTimestampClock) throws Exception + { + RelationalIngestor ingestor = RelationalIngestor.builder() + .ingestMode(ingestMode) + .relationalSink(BigQuerySink.get()) + .executionTimestampClock(executionTimestampClock) + .cleanupStagingData(options.cleanupStagingData()) + .collectStatistics(options.collectStatistics()) + .enableSchemaEvolution(options.enableSchemaEvolution()) + .schemaEvolutionCapabilitySet(Collections.emptySet()) + .caseConversion(CaseConversion.TO_UPPER) + .build(); + + IngestorResult result = ingestor.ingest(BigQueryConnection.of(getBigQueryConnection()), datasets); + + Map actualStats = result.statisticByName(); + + // Verify the database data + List> tableData = runQuery("select * from \"TEST\".\"MAIN\""); + + assertFileAndTableDataEquals(schema, expectedDataPath, tableData); + + // Verify statistics + Assertions.assertEquals(expectedStats.size(), actualStats.size()); + for (String statistic : expectedStats.keySet()) + { + Assertions.assertEquals(expectedStats.get(statistic).toString(), actualStats.get(StatisticName.valueOf(statistic)).toString()); + } + + // Return result (including updated datasets) + return result; + } + + protected IngestorResult executePlansAndVerifyResults(IngestMode ingestMode, PlannerOptions options, Datasets datasets, String[] schema, String expectedDataPath, Map expectedStats, Clock executionTimestampClock) throws Exception + { + return executePlansAndVerifyResults(ingestMode, options, datasets, schema, expectedDataPath, expectedStats, executionTimestampClock, Collections.emptySet()); + } + + protected IngestorResult executePlansAndVerifyResults(IngestMode ingestMode, PlannerOptions options, Datasets datasets, String[] schema, String expectedDataPath, Map expectedStats) throws Exception + { + return executePlansAndVerifyResults(ingestMode, options, datasets, schema, expectedDataPath, expectedStats, Clock.systemUTC()); + } + + protected IngestorResult executePlansAndVerifyResults(IngestMode ingestMode, PlannerOptions options, Datasets datasets, String[] schema, String expectedDataPath, Map expectedStats, Set userCapabilitySet) throws Exception + { + return executePlansAndVerifyResults(ingestMode, options, datasets, schema, expectedDataPath, expectedStats, Clock.systemUTC(), userCapabilitySet); + } + + protected IngestorResult executePlansAndVerifyResults(IngestMode ingestMode, PlannerOptions options, Datasets datasets, String[] schema, String expectedDataPath, Map expectedStats, Clock executionTimestampClock, Set userCapabilitySet) throws Exception + { + // Execute physical plans + RelationalIngestor ingestor = RelationalIngestor.builder() + .ingestMode(ingestMode) + .relationalSink(BigQuerySink.get()) + .executionTimestampClock(executionTimestampClock) + .cleanupStagingData(options.cleanupStagingData()) + .collectStatistics(options.collectStatistics()) + .enableSchemaEvolution(options.enableSchemaEvolution()) + .schemaEvolutionCapabilitySet(userCapabilitySet) + .build(); + IngestorResult result = ingestor.ingest(BigQueryConnection.of(getBigQueryConnection()), datasets); + + Map actualStats = result.statisticByName(); + + // Verify the database data + List> tableData = runQuery("select * from \"TEST\".\"main\""); + assertFileAndTableDataEquals(schema, expectedDataPath, tableData); + + // Verify statistics + Assertions.assertEquals(expectedStats.size(), actualStats.size()); + for (String statistic : expectedStats.keySet()) + { + Assertions.assertEquals(expectedStats.get(statistic).toString(), actualStats.get(StatisticName.valueOf(statistic)).toString()); + } + + // Return result (including updated datasets) + return result; + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BitempDeltaExecutorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BitempDeltaExecutorTest.java new file mode 100644 index 00000000000..8aae8ef73b6 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BitempDeltaExecutorTest.java @@ -0,0 +1,166 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.BitemporalDelta; +import org.finos.legend.engine.persistence.components.ingestmode.transactionmilestoning.BatchId; +import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.ValidDateTime; +import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.derivation.SourceSpecifiesFromDateTime; +import org.finos.legend.engine.persistence.components.relational.api.IngestorResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +import static org.finos.legend.engine.persistence.components.common.StatisticName.INCOMING_RECORD_COUNT; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_INSERTED; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_UPDATED; + +@Disabled +public class BitempDeltaExecutorTest extends BigQueryEndToEndTest +{ + /* + Scenario: Test milestoning Logic when staging table pre populated + */ + @Test + public void testMilestoning() throws Exception + { + BitemporalDelta ingestMode = BitemporalDelta.builder() + .digestField(digestName) + .transactionMilestoning(BatchId.builder() + .batchIdInName(batchIdInName) + .batchIdOutName(batchIdOutName) + .build()) + .validityMilestoning(ValidDateTime.builder() + .dateTimeFromName(startDateTimeName) + .dateTimeThruName(endDateTimeName) + .validityDerivation(SourceSpecifiesFromDateTime.builder() + .sourceDateTimeFromField(dateTimeName) + .build()) + .build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + delete("demo", "batch_metadata"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/bitemp_delta/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 1250000); + IngestorResult result = ingestViaExecutor(ingestMode, bitempStagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + String expectedPath = "src/test/resources/expected/bitemp_delta/data_pass1.csv"; + String[] schema = new String[] {indexName, balanceName, digestName, startDateTimeName, endDateTimeName, batchIdInName, batchIdOutName}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + long incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + long rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + Assertions.assertEquals(1, incomingRecords); + Assertions.assertEquals(1, rowsInserted); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/bitemp_delta/data_pass2.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 124000); + result = ingestViaExecutor(ingestMode, bitempStagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + long rowsUpdated = (long) result.statisticByName().get(ROWS_UPDATED); + Assertions.assertEquals(1, incomingRecords); + Assertions.assertEquals(1, rowsInserted); + Assertions.assertEquals(1, rowsUpdated); + + // Pass 3 + System.out.println("--------- Batch 3 started ------------"); + String pathPass3 = "src/test/resources/input/bitemp_delta/data_pass3.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 120000); + result = ingestViaExecutor(ingestMode, bitempStagingSchema, stagingFilter, pathPass3, fixedClock_2000_01_03); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass3.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + rowsUpdated = (long) result.statisticByName().get(ROWS_UPDATED); + Assertions.assertEquals(1, incomingRecords); + Assertions.assertEquals(1, rowsInserted); + Assertions.assertEquals(1, rowsUpdated); + + // Pass 4 + System.out.println("--------- Batch 4 started ------------"); + String pathPass4 = "src/test/resources/input/bitemp_delta/data_pass4.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 122000); + result = ingestViaExecutor(ingestMode, bitempStagingSchema, stagingFilter, pathPass4, fixedClock_2000_01_04); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass4.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + rowsUpdated = (long) result.statisticByName().get(ROWS_UPDATED); + Assertions.assertEquals(1, incomingRecords); + Assertions.assertEquals(1, rowsInserted); + Assertions.assertEquals(1, rowsUpdated); + + // Pass 5 + System.out.println("--------- Batch 5 started ------------"); + String pathPass5 = "src/test/resources/input/bitemp_delta/data_pass5.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 110000); + result = ingestViaExecutor(ingestMode, bitempStagingSchema, stagingFilter, pathPass5, fixedClock_2000_01_05); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass5.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + rowsUpdated = (long) result.statisticByName().get(ROWS_UPDATED); + Assertions.assertEquals(1, incomingRecords); + Assertions.assertEquals(0, rowsInserted); + Assertions.assertEquals(1, rowsUpdated); + + // Pass 6 + System.out.println("--------- Batch 6 started ------------"); + String pathPass6 = "src/test/resources/input/bitemp_delta/data_pass6.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 110000); + result = ingestViaExecutor(ingestMode, bitempStagingSchema, stagingFilter, pathPass6, fixedClock_2000_01_06); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass6.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + rowsUpdated = (long) result.statisticByName().get(ROWS_UPDATED); + Assertions.assertEquals(1, incomingRecords); + Assertions.assertEquals(0, rowsInserted); + Assertions.assertEquals(1, rowsUpdated); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BitempDeltaGeneratorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BitempDeltaGeneratorTest.java new file mode 100644 index 00000000000..b7c0dd61dfd --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BitempDeltaGeneratorTest.java @@ -0,0 +1,125 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.BitemporalDelta; +import org.finos.legend.engine.persistence.components.ingestmode.transactionmilestoning.BatchId; +import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.ValidDateTime; +import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.derivation.SourceSpecifiesFromDateTime; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +@Disabled +public class BitempDeltaGeneratorTest extends BigQueryEndToEndTest +{ + /* + Scenario: Test milestoning Logic when staging table pre populated + */ + @Test + public void testMilestoning() throws Exception + { + BitemporalDelta ingestMode = BitemporalDelta.builder() + .digestField(digestName) + .transactionMilestoning(BatchId.builder() + .batchIdInName(batchIdInName) + .batchIdOutName(batchIdOutName) + .build()) + .validityMilestoning(ValidDateTime.builder() + .dateTimeFromName(startDateTimeName) + .dateTimeThruName(endDateTimeName) + .validityDerivation(SourceSpecifiesFromDateTime.builder() + .sourceDateTimeFromField(dateTimeName) + .build()) + .build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + delete("demo", "batch_metadata"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/bitemp_delta/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 1250000); + ingestViaGenerator(ingestMode, bitempStagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + String expectedPath = "src/test/resources/expected/bitemp_delta/data_pass1.csv"; + String[] schema = new String[] {indexName, balanceName, digestName, startDateTimeName, endDateTimeName, batchIdInName, batchIdOutName}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/bitemp_delta/data_pass2.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 124000); + ingestViaGenerator(ingestMode, bitempStagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 3 + System.out.println("--------- Batch 3 started ------------"); + String pathPass3 = "src/test/resources/input/bitemp_delta/data_pass3.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 120000); + ingestViaGenerator(ingestMode, bitempStagingSchema, stagingFilter, pathPass3, fixedClock_2000_01_03); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass3.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 4 + System.out.println("--------- Batch 4 started ------------"); + String pathPass4 = "src/test/resources/input/bitemp_delta/data_pass4.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 122000); + ingestViaGenerator(ingestMode, bitempStagingSchema, stagingFilter, pathPass4, fixedClock_2000_01_04); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass4.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 5 + System.out.println("--------- Batch 5 started ------------"); + String pathPass5 = "src/test/resources/input/bitemp_delta/data_pass5.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 110000); + ingestViaGenerator(ingestMode, bitempStagingSchema, stagingFilter, pathPass5, fixedClock_2000_01_05); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass5.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 6 + System.out.println("--------- Batch 6 started ------------"); + String pathPass6 = "src/test/resources/input/bitemp_delta/data_pass6.csv"; + stagingFilter = DatasetFilter.of(balanceName, FilterType.EQUAL_TO, 110000); + ingestViaGenerator(ingestMode, bitempStagingSchema, stagingFilter, pathPass6, fixedClock_2000_01_06); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by " + batchIdInName + ", " + batchIdOutName + ", " + balanceName + " asc"); + expectedPath = "src/test/resources/expected/bitemp_delta/data_pass6.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalDeltaExecutorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalDeltaExecutorTest.java new file mode 100644 index 00000000000..3cdc97b59a5 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalDeltaExecutorTest.java @@ -0,0 +1,76 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.NontemporalDelta; +import org.finos.legend.engine.persistence.components.ingestmode.audit.DateTimeAuditing; +import org.finos.legend.engine.persistence.components.relational.api.IngestorResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.finos.legend.engine.persistence.components.common.StatisticName.INCOMING_RECORD_COUNT; + +@Disabled +public class NonTemporalDeltaExecutorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + NontemporalDelta ingestMode = NontemporalDelta.builder() + .digestField("digest") + .auditing(DateTimeAuditing.builder().dateTimeField("audit_ts").build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + IngestorResult result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/nontemporal_delta/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts", "audit_ts"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + long incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + Assertions.assertEquals(3, incomingRecords); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc"); + expectedPath = "src/test/resources/expected/nontemporal_delta/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + Assertions.assertEquals(3, incomingRecords); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalDeltaGeneratorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalDeltaGeneratorTest.java new file mode 100644 index 00000000000..d55230ffeb4 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalDeltaGeneratorTest.java @@ -0,0 +1,67 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.NontemporalDelta; +import org.finos.legend.engine.persistence.components.ingestmode.audit.DateTimeAuditing; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@Disabled +public class NonTemporalDeltaGeneratorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + NontemporalDelta ingestMode = NontemporalDelta.builder() + .digestField("digest") + .auditing(DateTimeAuditing.builder().dateTimeField("audit_ts").build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/nontemporal_delta/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts", "audit_ts"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc"); + expectedPath = "src/test/resources/expected/nontemporal_delta/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalSnapshotExecutorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalSnapshotExecutorTest.java new file mode 100644 index 00000000000..f72294d6cb1 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalSnapshotExecutorTest.java @@ -0,0 +1,84 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.NontemporalSnapshot; +import org.finos.legend.engine.persistence.components.ingestmode.audit.NoAuditing; +import org.finos.legend.engine.persistence.components.relational.api.IngestorResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.finos.legend.engine.persistence.components.common.StatisticName.INCOMING_RECORD_COUNT; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_DELETED; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_INSERTED; + +@Disabled +public class NonTemporalSnapshotExecutorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + NontemporalSnapshot ingestMode = NontemporalSnapshot + .builder() + .auditing(NoAuditing.builder().build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + IngestorResult result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/nontemporal_snapshot/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + long incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + long rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + Assertions.assertEquals(3, incomingRecords); + Assertions.assertEquals(3, rowsInserted); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc"); + expectedPath = "src/test/resources/expected/nontemporal_snapshot/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + long rowsDeleted = (long) result.statisticByName().get(ROWS_DELETED); + Assertions.assertEquals(3, incomingRecords); + Assertions.assertEquals(3, rowsInserted); + Assertions.assertEquals(3, rowsDeleted); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalSnapshotGeneratorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalSnapshotGeneratorTest.java new file mode 100644 index 00000000000..84f38fd7180 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/NonTemporalSnapshotGeneratorTest.java @@ -0,0 +1,67 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.NontemporalSnapshot; +import org.finos.legend.engine.persistence.components.ingestmode.audit.NoAuditing; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@Disabled +public class NonTemporalSnapshotGeneratorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + NontemporalSnapshot ingestMode = NontemporalSnapshot + .builder() + .auditing(NoAuditing.builder().build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/nontemporal_snapshot/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc"); + expectedPath = "src/test/resources/expected/nontemporal_snapshot/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/SchemaEvolutionTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/SchemaEvolutionTest.java new file mode 100644 index 00000000000..9cc30d8a255 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/SchemaEvolutionTest.java @@ -0,0 +1,628 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import com.google.cloud.bigquery.BigQuery; +import org.finos.legend.engine.persistence.components.BaseTestUtils; +import org.finos.legend.engine.persistence.components.executor.Executor; +import org.finos.legend.engine.persistence.components.ingestmode.NontemporalSnapshot; +import org.finos.legend.engine.persistence.components.ingestmode.audit.NoAuditing; +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Create; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Drop; +import org.finos.legend.engine.persistence.components.logicalplan.operations.Insert; +import org.finos.legend.engine.persistence.components.logicalplan.values.FieldValue; +import org.finos.legend.engine.persistence.components.logicalplan.values.TabularValues; +import org.finos.legend.engine.persistence.components.logicalplan.values.Value; +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.relational.bigquery.executor.BigQueryConnection; +import org.finos.legend.engine.persistence.components.relational.bigquery.executor.BigQueryHelper; +import org.finos.legend.engine.persistence.components.relational.sql.TabularData; +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; +import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; +import org.finos.legend.engine.persistence.components.schemaevolution.IncompatibleSchemaChangeException; +import org.finos.legend.engine.persistence.components.schemaevolution.SchemaEvolution; +import org.finos.legend.engine.persistence.components.schemaevolution.SchemaEvolutionResult; +import org.finos.legend.engine.persistence.components.util.SchemaEvolutionCapability; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithAllColumns; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithAllColumnsFromDb; + +@Disabled +public class SchemaEvolutionTest extends BigQueryEndToEndTest +{ + protected final String datasetName = "demo"; + + @Test + public void testSchemaValidation() throws IOException + { + String tableName = "test_data_types_supported"; + DatasetDefinition dataset = DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name(tableName) + .alias(tableName) + .schema(schemaWithAllColumns) + .build(); + BigQuery bigquery = getBigQueryConnection(); + BigQueryHelper bigQueryHelper = BigQueryHelper.of(bigquery); + RelationalSink relationalSink = BigQuerySink.get(); + Executor relationalExecutor = relationalSink.getRelationalExecutor(BigQueryConnection.of(bigquery)); + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get()); + + dropTable(relationalExecutor, transformer, dataset); + createTable(relationalExecutor, transformer, dataset); + + relationalSink.validateMainDatasetSchemaFn().execute(relationalExecutor, bigQueryHelper, dataset); + Dataset datasetConstructedFromDb = relationalSink.constructDatasetFromDatabaseFn().execute(relationalExecutor, bigQueryHelper, tableName, datasetName, projectId); + relationalSink.validateMainDatasetSchemaFn().execute(relationalExecutor, bigQueryHelper, datasetConstructedFromDb); + Assertions.assertEquals(dataset.withSchema(schemaWithAllColumnsFromDb), datasetConstructedFromDb); + } + + @Test + public void testSchemaEvolution() throws IOException + { + List list = Arrays.asList( + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_int64") + .alias("tsm_int64") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colInt64.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric") + .alias("tsm_numeric") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumeric.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_with_precision") + .alias("tsm_numeric_with_precision") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumericWithPrecision.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_with_scale") + .alias("tsm_numeric_with_scale") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumericWithScale.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_with_less_scale") + .alias("tsm_numeric_with_less_scale") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumericWithScale.withName("col").withType(BaseTestUtils.colNumericWithScale.type().withLength(32).withScale(3))).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_float64") + .alias("tsm_float64") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colFloat64.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_string") + .alias("tsm_string") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colString.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_string_with_length") + .alias("tsm_string_with_length") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colStringWithLength.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_string_with_big_length") + .alias("tsm_string_with_big_length") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colStringWithLength.withName("col").withType(BaseTestUtils.colStringWithLength.type().withLength(100))).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_date") + .alias("tsm_date") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colDate.withName("col").withPrimaryKey(false)).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_datetime") + .alias("tsm_datetime") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colDatetime.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_time") + .alias("tsm_time") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colTime.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_timestamp") + .alias("tsm_timestamp") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colTimestamp.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_bool") + .alias("tsm_bool") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colBool.withName("col")).build()) + .build(), + DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_json") + .alias("tsm_json") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colJson.withName("col")).build()) + .build()); + + List> alterSqls = Arrays.asList( + Arrays.asList(), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_precision` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_scale` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_less_scale` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList(), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_int64` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_precision` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_scale` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_less_scale` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList(), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_int64` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList(), + Arrays.asList(), + Arrays.asList(), + Arrays.asList(), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_int64` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList(), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_precision` ALTER COLUMN `col` SET DATA TYPE NUMERIC(33,4)"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_less_scale` ALTER COLUMN `col` SET DATA TYPE NUMERIC(33,4)"), + Arrays.asList(), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_int64` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList(), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_precision` ALTER COLUMN `col` SET DATA TYPE NUMERIC(32,3)"), + Arrays.asList(), + Arrays.asList(), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_int64` ALTER COLUMN `col` SET DATA TYPE FLOAT64"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric` ALTER COLUMN `col` SET DATA TYPE FLOAT64"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_precision` ALTER COLUMN `col` SET DATA TYPE FLOAT64"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_scale` ALTER COLUMN `col` SET DATA TYPE FLOAT64"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_with_less_scale` ALTER COLUMN `col` SET DATA TYPE FLOAT64"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_string_with_length` ALTER COLUMN `col` SET DATA TYPE STRING"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_string_with_big_length` ALTER COLUMN `col` SET DATA TYPE STRING"), + Arrays.asList(), + Arrays.asList(), + Arrays.asList(), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_string_with_length` ALTER COLUMN `col` SET DATA TYPE STRING(100)"), + Arrays.asList() + ); + + List exceptionMessages = Arrays.asList( + "Breaking schema change from datatype \"STRING\" to \"INT64\"", + "Breaking schema change from datatype \"STRING\" to \"INT64\"", + "Breaking schema change from datatype \"STRING\" to \"INT64\"", + "Breaking schema change from datatype \"DATE\" to \"INT64\"", + "Breaking schema change from datatype \"DATETIME\" to \"INT64\"", + "Breaking schema change from datatype \"TIME\" to \"INT64\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"INT64\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"INT64\"", + "Breaking schema change from datatype \"JSON\" to \"INT64\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"DATE\" to \"NUMERIC\"", + "Breaking schema change from datatype \"DATETIME\" to \"NUMERIC\"", + "Breaking schema change from datatype \"TIME\" to \"NUMERIC\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"NUMERIC\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"NUMERIC\"", + "Breaking schema change from datatype \"JSON\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"DATE\" to \"NUMERIC\"", + "Breaking schema change from datatype \"DATETIME\" to \"NUMERIC\"", + "Breaking schema change from datatype \"TIME\" to \"NUMERIC\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"NUMERIC\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"NUMERIC\"", + "Breaking schema change from datatype \"JSON\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"DATE\" to \"NUMERIC\"", + "Breaking schema change from datatype \"DATETIME\" to \"NUMERIC\"", + "Breaking schema change from datatype \"TIME\" to \"NUMERIC\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"NUMERIC\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"NUMERIC\"", + "Breaking schema change from datatype \"JSON\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"NUMERIC\"", + "Breaking schema change from datatype \"DATE\" to \"NUMERIC\"", + "Breaking schema change from datatype \"DATETIME\" to \"NUMERIC\"", + "Breaking schema change from datatype \"TIME\" to \"NUMERIC\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"NUMERIC\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"NUMERIC\"", + "Breaking schema change from datatype \"JSON\" to \"NUMERIC\"", + "Breaking schema change from datatype \"STRING\" to \"FLOAT64\"", + "Breaking schema change from datatype \"STRING\" to \"FLOAT64\"", + "Breaking schema change from datatype \"STRING\" to \"FLOAT64\"", + "Breaking schema change from datatype \"DATE\" to \"FLOAT64\"", + "Breaking schema change from datatype \"DATETIME\" to \"FLOAT64\"", + "Breaking schema change from datatype \"TIME\" to \"FLOAT64\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"FLOAT64\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"FLOAT64\"", + "Breaking schema change from datatype \"JSON\" to \"FLOAT64\"", + "Breaking schema change from datatype \"INTEGER\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"FLOAT\" to \"STRING\"", + "Breaking schema change from datatype \"DATE\" to \"STRING\"", + "Breaking schema change from datatype \"DATETIME\" to \"STRING\"", + "Breaking schema change from datatype \"TIME\" to \"STRING\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"STRING\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"STRING\"", + "Breaking schema change from datatype \"JSON\" to \"STRING\"", + "Breaking schema change from datatype \"INTEGER\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"FLOAT\" to \"STRING\"", + "Breaking schema change from datatype \"DATE\" to \"STRING\"", + "Breaking schema change from datatype \"DATETIME\" to \"STRING\"", + "Breaking schema change from datatype \"TIME\" to \"STRING\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"STRING\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"STRING\"", + "Breaking schema change from datatype \"JSON\" to \"STRING\"", + "Breaking schema change from datatype \"INTEGER\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"NUMERIC\" to \"STRING\"", + "Breaking schema change from datatype \"FLOAT\" to \"STRING\"", + "Breaking schema change from datatype \"DATE\" to \"STRING\"", + "Breaking schema change from datatype \"DATETIME\" to \"STRING\"", + "Breaking schema change from datatype \"TIME\" to \"STRING\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"STRING\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"STRING\"", + "Breaking schema change from datatype \"JSON\" to \"STRING\"", + "Breaking schema change from datatype \"INTEGER\" to \"DATE\"", + "Breaking schema change from datatype \"NUMERIC\" to \"DATE\"", + "Breaking schema change from datatype \"NUMERIC\" to \"DATE\"", + "Breaking schema change from datatype \"NUMERIC\" to \"DATE\"", + "Breaking schema change from datatype \"NUMERIC\" to \"DATE\"", + "Breaking schema change from datatype \"FLOAT\" to \"DATE\"", + "Breaking schema change from datatype \"STRING\" to \"DATE\"", + "Breaking schema change from datatype \"STRING\" to \"DATE\"", + "Breaking schema change from datatype \"STRING\" to \"DATE\"", + "Breaking schema change from datatype \"TIME\" to \"DATE\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"DATE\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"DATE\"", + "Breaking schema change from datatype \"JSON\" to \"DATE\"", + "Breaking schema change from datatype \"INTEGER\" to \"DATETIME\"", + "Breaking schema change from datatype \"NUMERIC\" to \"DATETIME\"", + "Breaking schema change from datatype \"NUMERIC\" to \"DATETIME\"", + "Breaking schema change from datatype \"NUMERIC\" to \"DATETIME\"", + "Breaking schema change from datatype \"NUMERIC\" to \"DATETIME\"", + "Breaking schema change from datatype \"FLOAT\" to \"DATETIME\"", + "Breaking schema change from datatype \"STRING\" to \"DATETIME\"", + "Breaking schema change from datatype \"STRING\" to \"DATETIME\"", + "Breaking schema change from datatype \"STRING\" to \"DATETIME\"", + "Breaking schema change from datatype \"DATE\" to \"DATETIME\"", + "Breaking schema change from datatype \"TIME\" to \"DATETIME\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"DATETIME\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"DATETIME\"", + "Breaking schema change from datatype \"JSON\" to \"DATETIME\"", + "Breaking schema change from datatype \"INTEGER\" to \"TIME\"", + "Breaking schema change from datatype \"NUMERIC\" to \"TIME\"", + "Breaking schema change from datatype \"NUMERIC\" to \"TIME\"", + "Breaking schema change from datatype \"NUMERIC\" to \"TIME\"", + "Breaking schema change from datatype \"NUMERIC\" to \"TIME\"", + "Breaking schema change from datatype \"FLOAT\" to \"TIME\"", + "Breaking schema change from datatype \"STRING\" to \"TIME\"", + "Breaking schema change from datatype \"STRING\" to \"TIME\"", + "Breaking schema change from datatype \"STRING\" to \"TIME\"", + "Breaking schema change from datatype \"DATE\" to \"TIME\"", + "Breaking schema change from datatype \"DATETIME\" to \"TIME\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"TIME\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"TIME\"", + "Breaking schema change from datatype \"JSON\" to \"TIME\"", + "Breaking schema change from datatype \"INTEGER\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"NUMERIC\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"NUMERIC\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"NUMERIC\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"NUMERIC\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"FLOAT\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"STRING\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"STRING\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"STRING\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"DATE\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"DATETIME\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"TIME\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"JSON\" to \"TIMESTAMP\"", + "Breaking schema change from datatype \"INTEGER\" to \"BOOL\"", + "Breaking schema change from datatype \"NUMERIC\" to \"BOOL\"", + "Breaking schema change from datatype \"NUMERIC\" to \"BOOL\"", + "Breaking schema change from datatype \"NUMERIC\" to \"BOOL\"", + "Breaking schema change from datatype \"NUMERIC\" to \"BOOL\"", + "Breaking schema change from datatype \"FLOAT\" to \"BOOL\"", + "Breaking schema change from datatype \"STRING\" to \"BOOL\"", + "Breaking schema change from datatype \"STRING\" to \"BOOL\"", + "Breaking schema change from datatype \"STRING\" to \"BOOL\"", + "Breaking schema change from datatype \"DATE\" to \"BOOL\"", + "Breaking schema change from datatype \"DATETIME\" to \"BOOL\"", + "Breaking schema change from datatype \"TIME\" to \"BOOL\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"BOOL\"", + "Breaking schema change from datatype \"JSON\" to \"BOOL\"", + "Breaking schema change from datatype \"INTEGER\" to \"JSON\"", + "Breaking schema change from datatype \"NUMERIC\" to \"JSON\"", + "Breaking schema change from datatype \"NUMERIC\" to \"JSON\"", + "Breaking schema change from datatype \"NUMERIC\" to \"JSON\"", + "Breaking schema change from datatype \"NUMERIC\" to \"JSON\"", + "Breaking schema change from datatype \"FLOAT\" to \"JSON\"", + "Breaking schema change from datatype \"STRING\" to \"JSON\"", + "Breaking schema change from datatype \"STRING\" to \"JSON\"", + "Breaking schema change from datatype \"STRING\" to \"JSON\"", + "Breaking schema change from datatype \"DATE\" to \"JSON\"", + "Breaking schema change from datatype \"DATETIME\" to \"JSON\"", + "Breaking schema change from datatype \"TIME\" to \"JSON\"", + "Breaking schema change from datatype \"TIMESTAMP\" to \"JSON\"", + "Breaking schema change from datatype \"BOOLEAN\" to \"JSON\"" + ); + + BigQuery bigquery = getBigQueryConnection(); + RelationalSink relationalSink = BigQuerySink.get(); + BigQueryHelper bigQueryHelper = BigQueryHelper.of(bigquery); + Executor relationalExecutor = relationalSink.getRelationalExecutor(BigQueryConnection.of(bigquery)); + RelationalTransformer transformer = new RelationalTransformer(relationalSink); + + int size = list.size(); + int alterCallIndex = 0; + int exceptionMessageIndex = 0; + for (int stage = 0; stage < size; stage++) + { + for (int main = 0; main < size; main++) + { + if (stage == main) + { + continue; + } + SchemaEvolution schemaEvolution = new SchemaEvolution(relationalSink, NontemporalSnapshot.builder().auditing(NoAuditing.builder().build()).build(), Arrays.stream(SchemaEvolutionCapability.values()).collect(Collectors.toSet())); + DatasetDefinition datasetDefinitionStage = list.get(stage); + DatasetDefinition datasetDefinitionMain = list.get(main); + refreshDataset(relationalExecutor, transformer, datasetDefinitionMain, null); + Dataset datasetMain = relationalSink.constructDatasetFromDatabaseFn().execute(relationalExecutor, bigQueryHelper, datasetDefinitionMain.name(), datasetName, projectId); + FieldType typeStage = datasetDefinitionStage.schema().fields().get(0).type(); + FieldType typeMain = datasetMain.schema().fields().get(0).type(); + DataType dataTypeStage = typeStage.dataType(); + DataType dataTypeMain = typeMain.dataType(); + if (typeMain.equals(typeStage)) + { + //assert no change + schemaEvolve(relationalExecutor, transformer, schemaEvolution, datasetMain, datasetDefinitionStage, datasetMain, alterSqls.get(alterCallIndex++)); + } + else + { + FieldType typeToAssert; + Optional lengthToAssert = typeMain.length(); + if (!typeStage.length().isPresent() || lengthToAssert.isPresent() && typeStage.length().get() > lengthToAssert.get()) + { + lengthToAssert = typeStage.length(); + } + + Optional scaleToAssert = typeMain.scale(); + if (typeStage.scale().isPresent() && (!scaleToAssert.isPresent() || typeStage.scale().get() > scaleToAssert.get())) + { + scaleToAssert = typeStage.scale(); + } + if (!lengthToAssert.isPresent()) + { + scaleToAssert = Optional.empty(); + } + + if (relationalSink.supportsImplicitMapping(dataTypeMain, dataTypeStage)) + { + //assert no changes + typeToAssert = typeMain.withLength(lengthToAssert).withScale(scaleToAssert); + Dataset datasetToAssert = datasetMain.withSchema(datasetMain.schema().withFields(datasetMain.schema().fields().get(0).withType(typeToAssert))); + schemaEvolve(relationalExecutor, transformer, schemaEvolution, datasetMain, datasetDefinitionStage, datasetToAssert, alterSqls.get(alterCallIndex++)); + } + else if (relationalSink.supportsExplicitMapping(dataTypeMain, dataTypeStage)) + { + //assert stage schema + typeToAssert = typeStage.withLength(lengthToAssert).withScale(scaleToAssert); + Dataset datasetToAssert = datasetMain.withSchema(datasetDefinitionStage.schema().withFields(datasetDefinitionStage.schema().fields().get(0).withType(typeToAssert))); + schemaEvolve(relationalExecutor, transformer, schemaEvolution, datasetMain, datasetDefinitionStage, datasetToAssert, alterSqls.get(alterCallIndex++)); + } + else + { + try + { + schemaEvolve(relationalExecutor, transformer, schemaEvolution, datasetMain, datasetDefinitionStage, null, null); + } + catch (Exception e) + { + Assertions.assertTrue(e instanceof IncompatibleSchemaChangeException); + Assertions.assertEquals(exceptionMessages.get(exceptionMessageIndex++), e.getMessage()); + } + } + } + } + } + } + + @Test + public void testLengthEvolution() throws IOException + { + DatasetDefinition numeric1 = DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_1") + .alias("tsm_numeric_1") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumericWithScale.withName("col") + .withType(BaseTestUtils.colNumericWithScale.type().withLength(20).withScale(3))).build()) + .build(); + DatasetDefinition numeric2 = DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_2") + .alias("tsm_numeric_2") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumericWithScale.withName("col") + .withType(BaseTestUtils.colNumericWithScale.type().withLength(20).withScale(5))).build()) + .build(); + DatasetDefinition numeric3 = DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_3") + .alias("tsm_numeric_3") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumericWithScale.withName("col") + .withType(BaseTestUtils.colNumericWithScale.type().withLength(22).withScale(5))).build()) + .build(); + DatasetDefinition numeric4 = DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_4") + .alias("tsm_numeric_4") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumericWithPrecision.withName("col")).build()) + .build(); + DatasetDefinition numeric5 = DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_5") + .alias("tsm_numeric_5") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumericWithScale.withName("col") + .withType(BaseTestUtils.colNumericWithScale.type().withLength(32).withScale(3))).build()) + .build(); + DatasetDefinition numeric6 = DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_6") + .alias("tsm_numeric_6") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumericWithScale.withName("col") + .withType(BaseTestUtils.colNumericWithScale.type().withLength(34).withScale(5))).build()) + .build(); + DatasetDefinition numeric7 = DatasetDefinition.builder() + .database(projectId) + .group(datasetName) + .name("tsm_numeric_7") + .alias("tsm_numeric_7") + .schema(SchemaDefinition.builder().addFields(BaseTestUtils.colNumeric.withName("col")).build()) + .build(); + + List stageDefinition = Arrays.asList( + numeric1, numeric2, numeric3, numeric3, numeric1, numeric4, numeric7, numeric7, numeric7); + List mainDefinition = Arrays.asList( + numeric2, numeric1, numeric1, numeric2, numeric4, numeric2, numeric1, numeric4, numeric6); + List assertionDefinition = Arrays.asList( + numeric3, numeric3, numeric3, numeric3, numeric5, numeric6, numeric7, numeric7, numeric7); + List> alterSqlToAssert = Arrays.asList( + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_2` ALTER COLUMN `col` SET DATA TYPE NUMERIC(22,5)"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_1` ALTER COLUMN `col` SET DATA TYPE NUMERIC(22,5)"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_1` ALTER COLUMN `col` SET DATA TYPE NUMERIC(22,5)"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_2` ALTER COLUMN `col` SET DATA TYPE NUMERIC(22,5)"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_4` ALTER COLUMN `col` SET DATA TYPE NUMERIC(32,3)"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_2` ALTER COLUMN `col` SET DATA TYPE NUMERIC(34,5)"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_1` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_4` ALTER COLUMN `col` SET DATA TYPE NUMERIC"), + Arrays.asList("ALTER TABLE `" + projectId + "`.`" + datasetName + "`.`tsm_numeric_6` ALTER COLUMN `col` SET DATA TYPE NUMERIC")); + + int size = stageDefinition.size(); + for (int i = 0; i < size; i++) + { + BigQuery bigquery = getBigQueryConnection(); + RelationalSink relationalSink = BigQuerySink.get(); + Executor relationalExecutor = relationalSink.getRelationalExecutor(BigQueryConnection.of(bigquery)); + RelationalTransformer transformer = new RelationalTransformer(relationalSink); + SchemaEvolution schemaEvolution = new SchemaEvolution(relationalSink, NontemporalSnapshot.builder().auditing(NoAuditing.builder().build()).build(), Arrays.stream(SchemaEvolutionCapability.values()).collect(Collectors.toSet())); + DatasetDefinition datasetDefinitionStage = stageDefinition.get(i); + DatasetDefinition datasetDefinitionMain = mainDefinition.get(i); + DatasetDefinition datasetDefinitionAssert = assertionDefinition.get(i); + refreshDataset(relationalExecutor, transformer, datasetDefinitionMain, null); + schemaEvolve(relationalExecutor, transformer, schemaEvolution, datasetDefinitionMain, datasetDefinitionStage, datasetDefinitionAssert.withName(datasetDefinitionMain.name()).withAlias(datasetDefinitionMain.alias()), alterSqlToAssert.get(i)); + } + } + + private static void schemaEvolve(Executor relationalExecutor, RelationalTransformer transformer, SchemaEvolution schemaEvolution, Dataset datasetMain, Dataset datasetStage, Dataset datasetToAssert, List alterSqlsToAssert) + { + SchemaEvolutionResult schemaEvolutionResult = schemaEvolution.buildLogicalPlanForSchemaEvolution(datasetMain, datasetStage); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(schemaEvolutionResult.logicalPlan()); + Assertions.assertEquals(alterSqlsToAssert, physicalPlan.getSqlList()); + relationalExecutor.executePhysicalPlan(physicalPlan); + Assertions.assertEquals(datasetToAssert, schemaEvolutionResult.evolvedDataset()); + } + + private static void refreshDataset(Executor relationalExecutor, RelationalTransformer transformer, DatasetDefinition datasetDefinition, Value value) + { + dropTable(relationalExecutor, transformer, datasetDefinition); + createTable(relationalExecutor, transformer, datasetDefinition); + if (value != null) + { + insertValues(relationalExecutor, transformer, datasetDefinition, value); + } + } + + private static void insertValues(Executor relationalExecutor, RelationalTransformer transformer, DatasetDefinition datasetDefinition, Value value) + { + Dataset values = TabularValues.builder().addValues(Collections.singletonList(value)).columnCount(1).build(); + List fields = Collections.singletonList(FieldValue.builder().fieldName(datasetDefinition.schema().fields().get(0).name()).build()); + LogicalPlan insertValuesLogicalPlan = LogicalPlan.builder().addOps(Insert.of(datasetDefinition, values, fields)).build(); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(insertValuesLogicalPlan); + relationalExecutor.executePhysicalPlan(physicalPlan); + } + + private static void createTable(Executor relationalExecutor, RelationalTransformer transformer, DatasetDefinition datasetDefinition) + { + LogicalPlan createTableLogicalPlan = LogicalPlan.builder().addOps(Create.of(false, datasetDefinition)).build(); + relationalExecutor.executePhysicalPlan(transformer.generatePhysicalPlan(createTableLogicalPlan)); + } + + private static void dropTable(Executor relationalExecutor, RelationalTransformer transformer, DatasetDefinition datasetDefinition) + { + LogicalPlan dropTableLogicalPlan = LogicalPlan.builder().addOps(Drop.of(true, datasetDefinition, false)).build(); + relationalExecutor.executePhysicalPlan(transformer.generatePhysicalPlan(dropTableLogicalPlan)); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempDeltaExecutorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempDeltaExecutorTest.java new file mode 100644 index 00000000000..ea55bf64817 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempDeltaExecutorTest.java @@ -0,0 +1,88 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.UnitemporalDelta; +import org.finos.legend.engine.persistence.components.ingestmode.transactionmilestoning.BatchId; +import org.finos.legend.engine.persistence.components.relational.api.IngestorResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.finos.legend.engine.persistence.components.common.StatisticName.INCOMING_RECORD_COUNT; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_INSERTED; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_UPDATED; + +@Disabled +public class UnitempDeltaExecutorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + UnitemporalDelta ingestMode = UnitemporalDelta.builder() + .digestField("digest") + .transactionMilestoning(BatchId.builder() + .batchIdInName("batch_id_in") + .batchIdOutName("batch_id_out") + .build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + delete("demo", "batch_metadata"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + IngestorResult result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/unitemp_delta/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts", "batch_id_in", "batch_id_out"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + long incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + long rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + Assertions.assertEquals(3, incomingRecords); + Assertions.assertEquals(3, rowsInserted); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc, insert_ts"); + expectedPath = "src/test/resources/expected/unitemp_delta/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + long rowsUpdated = (long) result.statisticByName().get(ROWS_UPDATED); + Assertions.assertEquals(3, incomingRecords); + Assertions.assertEquals(1, rowsInserted); + Assertions.assertEquals(1, rowsUpdated); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempDeltaGeneratorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempDeltaGeneratorTest.java new file mode 100644 index 00000000000..e6b1e4a78eb --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempDeltaGeneratorTest.java @@ -0,0 +1,72 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.UnitemporalDelta; +import org.finos.legend.engine.persistence.components.ingestmode.transactionmilestoning.BatchId; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@Disabled +public class UnitempDeltaGeneratorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + UnitemporalDelta ingestMode = UnitemporalDelta.builder() + .digestField("digest") + .transactionMilestoning(BatchId.builder() + .batchIdInName("batch_id_in") + .batchIdOutName("batch_id_out") + .build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + delete("demo", "batch_metadata"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/unitemp_delta/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts", "batch_id_in", "batch_id_out"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc, insert_ts"); + expectedPath = "src/test/resources/expected/unitemp_delta/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + } + +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempSnapshotExecutorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempSnapshotExecutorTest.java new file mode 100644 index 00000000000..d85ec0e4ede --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempSnapshotExecutorTest.java @@ -0,0 +1,93 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.UnitemporalSnapshot; +import org.finos.legend.engine.persistence.components.ingestmode.transactionmilestoning.BatchId; +import org.finos.legend.engine.persistence.components.relational.api.IngestorResult; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.finos.legend.engine.persistence.components.common.StatisticName.INCOMING_RECORD_COUNT; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_INSERTED; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_TERMINATED; +import static org.finos.legend.engine.persistence.components.common.StatisticName.ROWS_UPDATED; + +@Disabled +public class UnitempSnapshotExecutorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + UnitemporalSnapshot ingestMode = UnitemporalSnapshot.builder() + .digestField("digest") + .transactionMilestoning(BatchId.builder() + .batchIdInName("batch_id_in") + .batchIdOutName("batch_id_out") + .build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + delete("demo", "batch_metadata"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + IngestorResult result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/unitemp_snapshot/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts", "batch_id_in", "batch_id_out"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + long incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + long rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + Assertions.assertEquals(3, incomingRecords); + Assertions.assertEquals(3, rowsInserted); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + result = ingestViaExecutor(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc, insert_ts"); + expectedPath = "src/test/resources/expected/unitemp_snapshot/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + incomingRecords = (long) result.statisticByName().get(INCOMING_RECORD_COUNT); + rowsInserted = (long) result.statisticByName().get(ROWS_INSERTED); + long rowsUpdated = (long) result.statisticByName().get(ROWS_UPDATED); + long rowsTerminated = (long) result.statisticByName().get(ROWS_TERMINATED); + + Assertions.assertEquals(3, incomingRecords); + Assertions.assertEquals(1, rowsInserted); + Assertions.assertEquals(1, rowsUpdated); + Assertions.assertEquals(1, rowsTerminated); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempSnapshotGeneratorTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempSnapshotGeneratorTest.java new file mode 100644 index 00000000000..452f6362147 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/UnitempSnapshotGeneratorTest.java @@ -0,0 +1,73 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.e2e; + +import org.finos.legend.engine.persistence.components.common.DatasetFilter; +import org.finos.legend.engine.persistence.components.common.FilterType; +import org.finos.legend.engine.persistence.components.ingestmode.UnitemporalSnapshot; +import org.finos.legend.engine.persistence.components.ingestmode.transactionmilestoning.BatchId; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +@Disabled +public class UnitempSnapshotGeneratorTest extends BigQueryEndToEndTest +{ + + @Test + public void testMilestoning() throws IOException, InterruptedException + { + + UnitemporalSnapshot ingestMode = UnitemporalSnapshot.builder() + .digestField("digest") + .transactionMilestoning(BatchId.builder() + .batchIdInName("batch_id_in") + .batchIdOutName("batch_id_out") + .build()) + .build(); + + // Clean up + delete("demo", "main"); + delete("demo", "staging"); + delete("demo", "batch_metadata"); + + // Pass 1 + System.out.println("--------- Batch 1 started ------------"); + String pathPass1 = "src/test/resources/input/data_pass1.csv"; + DatasetFilter stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-01 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass1, fixedClock_2000_01_01); + + // Verify + List> tableData = runQuery("select * from `demo`.`main` order by id asc"); + String expectedPath = "src/test/resources/expected/unitemp_snapshot/data_pass1.csv"; + String [] schema = new String[] {"id", "name", "amount", "biz_date", "digest", "insert_ts", "batch_id_in", "batch_id_out"}; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + + // Pass 2 + System.out.println("--------- Batch 2 started ------------"); + String pathPass2 = "src/test/resources/input/data_pass2.csv"; + stagingFilter = DatasetFilter.of("insert_ts", FilterType.EQUAL_TO, "2023-01-02 00:00:00"); + ingestViaGenerator(ingestMode, stagingSchema, stagingFilter, pathPass2, fixedClock_2000_01_02); + + // Verify + tableData = runQuery("select * from `demo`.`main` order by id asc, insert_ts"); + expectedPath = "src/test/resources/expected/unitemp_snapshot/data_pass2.csv"; + assertFileAndTableDataEquals(schema, expectedPath, tableData); + } + +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/AppendOnlyTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/AppendOnlyTest.java new file mode 100644 index 00000000000..9798e8f8dfc --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/AppendOnlyTest.java @@ -0,0 +1,258 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.common.StatisticName; +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.DataSplitRange; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.junit.jupiter.api.Assertions; + +import java.util.ArrayList; +import java.util.List; + +public class AppendOnlyTest extends org.finos.legend.engine.persistence.components.ingestmode.nontemporal.AppendOnlyTest +{ + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT 0 as `rowsUpdated`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT COUNT(*) as `rowsInserted` FROM `mydb`.`staging` as stage"; + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + @Override + public void verifyAppendOnlyAllowDuplicatesNoAuditing(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`, `digest`) " + + "(SELECT * FROM `mydb`.`staging` as stage)"; + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTableCreateQueryWithNoPKs, preActionsSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(0)); + + // Stats + verifyStats(operations); + } + + @Override + public void verifyAppendOnlyAllowDuplicatesWithAuditing(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_update_time`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "FROM `mydb`.`staging` as stage)"; + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTableCreateQueryWithAuditAndNoPKs, preActionsSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(0)); + + // Stats + verifyStats(operations); + } + + @Override + public void verifyAppendOnlyAllowDuplicatesWithAuditingWithDataSplits(List generatorResults, List dataSplitRanges) + { + String insertSql = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_update_time`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "FROM `mydb`.`staging` as stage " + + "WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestPlusUpdateTimestampCreateQuery, generatorResults.get(0).preActionsSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(insertSql, dataSplitRanges.get(0)), generatorResults.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(insertSql, dataSplitRanges.get(1)), generatorResults.get(1).ingestSql().get(0)); + Assertions.assertEquals(2, generatorResults.size()); + + // Stats + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage " + + "WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + String rowsInserted = "SELECT COUNT(*) as `rowsInserted` FROM `mydb`.`staging` as stage " + + "WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + + Assertions.assertEquals(enrichSqlWithDataSplits(incomingRecordCount, dataSplitRanges.get(0)), generatorResults.get(0).postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(enrichSqlWithDataSplits(incomingRecordCount, dataSplitRanges.get(1)), generatorResults.get(1).postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsUpdated, generatorResults.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_UPDATED)); + Assertions.assertEquals(rowsDeleted, generatorResults.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + Assertions.assertEquals(enrichSqlWithDataSplits(rowsInserted, dataSplitRanges.get(0)), generatorResults.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_INSERTED)); + Assertions.assertEquals(rowsTerminated, generatorResults.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + } + + @Override + public void verifyAppendOnlyFailOnDuplicatesNoAuditing(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`, `digest`) " + + "(SELECT * FROM `mydb`.`staging` as stage)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(0)); + + // Stats + verifyStats(operations); + } + + @Override + public void verifyAppendOnlyFailOnDuplicatesWithAuditing(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `batch_update_time`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "FROM `mydb`.`staging` as stage)"; + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTableWithAuditNotPKCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(0)); + + // Stats + verifyStats(operations); + } + + @Override + public void verifyAppendOnlyFilterDuplicatesNoAuditing(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`) " + + "(SELECT * FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` = stage.`digest`))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(0)); + + // Stats + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsUpdated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_UPDATED)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyAppendOnlyFilterDuplicatesWithAuditing(GeneratorResult queries) + { + List preActionsSqlList = queries.preActionsSql(); + List milestoningSqlList = queries.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`, `digest`, `batch_update_time`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE ((sink.`id` = stage.`id`) AND " + + "(sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`))))"; + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestPlusUpdateTimestampCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(0)); + + List postActionsSql = queries.postActionsSql(); + List expectedSQL = new ArrayList<>(); + expectedSQL.add(BigQueryTestArtifacts.expectedStagingCleanupQuery); + assertIfListsAreSameIgnoringOrder(expectedSQL, postActionsSql); + + // Stats + String rowsInserted = "SELECT COUNT(*) as `rowsInserted` FROM `mydb`.`main` as sink WHERE sink.`batch_update_time` = (SELECT MAX(sink.`batch_update_time`) FROM `mydb`.`main` as sink)"; + Assertions.assertEquals(incomingRecordCount, queries.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsUpdated, queries.postIngestStatisticsSql().get(StatisticName.ROWS_UPDATED)); + Assertions.assertEquals(rowsDeleted, queries.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + Assertions.assertEquals(rowsInserted, queries.postIngestStatisticsSql().get(StatisticName.ROWS_INSERTED)); + Assertions.assertEquals(rowsTerminated, queries.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + } + + @Override + public void verifyAppendOnlyFilterDuplicatesWithAuditingWithDataSplit(List operations, List dataSplitRanges) + { + String insertSql = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_update_time`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND " + + "(NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` = stage.`digest`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestPlusUpdateTimestampCreateQuery, operations.get(0).preActionsSql().get(0)); + + Assertions.assertEquals(enrichSqlWithDataSplits(insertSql, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(insertSql, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + // Stats + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage " + + "WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + String rowsInserted = "SELECT COUNT(*) as `rowsInserted` FROM `mydb`.`main` as sink WHERE sink.`batch_update_time` = (SELECT MAX(sink.`batch_update_time`) FROM `mydb`.`main` as sink)"; + + Assertions.assertEquals(enrichSqlWithDataSplits(incomingRecordCount, dataSplitRanges.get(0)), operations.get(0).postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(enrichSqlWithDataSplits(incomingRecordCount, dataSplitRanges.get(1)), operations.get(1).postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsUpdated, operations.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_UPDATED)); + Assertions.assertEquals(rowsDeleted, operations.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + Assertions.assertEquals(rowsInserted, operations.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_INSERTED)); + Assertions.assertEquals(rowsTerminated, operations.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + } + + @Override + public void verifyAppendOnlyWithUpperCaseOptimizer(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `MYDB`.`MAIN` (`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`) " + + "(SELECT * FROM `MYDB`.`STAGING` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `MYDB`.`MAIN` as sink " + + "WHERE ((sink.`ID` = stage.`ID`) " + + "AND (sink.`NAME` = stage.`NAME`)) " + + "AND (sink.`DIGEST` = stage.`DIGEST`))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQueryWithUpperCase, preActionsSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(0)); + } + + @Override + public void verifyAppendOnlyWithLessColumnsInStaging(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`) " + + "(SELECT * FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` = stage.`digest`))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(0)); + } + + private void verifyStats(GeneratorResult operations) + { + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsUpdated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_UPDATED)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + Assertions.assertEquals(rowsInserted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_INSERTED)); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BigQueryTestArtifacts.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BigQueryTestArtifacts.java new file mode 100644 index 00000000000..830e48f4440 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BigQueryTestArtifacts.java @@ -0,0 +1,343 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +public class BigQueryTestArtifacts +{ + public static String expectedBaseTableCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "PRIMARY KEY (`id`, `name`) NOT ENFORCED)"; + + public static String expectedBaseTableCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + + "(`ID` INT64 NOT NULL," + + "`NAME` STRING NOT NULL," + + "`AMOUNT` FLOAT64," + + "`BIZ_DATE` DATE," + + "PRIMARY KEY (`ID`, `NAME`) NOT ENFORCED)"; + + public static String expectedBaseTablePlusDigestCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`digest` STRING," + + "PRIMARY KEY (`id`, `name`) NOT ENFORCED)"; + + public static String expectedBaseTablePlusDigestPlusVersionCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`digest` STRING," + + "`version` INT64," + + "PRIMARY KEY (`id`, `name`) NOT ENFORCED)"; + + public static String expectedBaseTablePlusDigestPlusVersionCreateQueryUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`(" + + "`ID` INT64 NOT NULL," + + "`NAME` STRING NOT NULL," + + "`AMOUNT` FLOAT64," + + "`BIZ_DATE` DATE," + + "`DIGEST` STRING," + + "`VERSION` INT64," + + "PRIMARY KEY (`ID`, `NAME`) NOT ENFORCED)"; + + public static String expectedBaseTablePlusDigestCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`(" + + "`ID` INT64 NOT NULL," + + "`NAME` STRING NOT NULL," + + "`AMOUNT` FLOAT64," + + "`BIZ_DATE` DATE," + + "`DIGEST` STRING," + + "PRIMARY KEY (`ID`, `NAME`) NOT ENFORCED)"; + + public static String expectedBaseTableCreateQueryWithNoPKs = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64," + + "`name` STRING," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`digest` STRING)"; + + public static String expectedBaseTableCreateQueryWithAuditAndNoPKs = "CREATE TABLE IF NOT EXISTS `mydb`.`main`" + + "(`id` INT64,`name` STRING,`amount` FLOAT64,`biz_date` DATE,`digest` STRING,`batch_update_time` DATETIME)"; + + public static String expectedMainTableBatchIdAndVersionBasedCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL,`name` STRING NOT NULL,`amount` FLOAT64,`biz_date` DATE,`digest` STRING,`version` INT64," + + "`batch_id_in` INT64 NOT NULL,`batch_id_out` INT64,PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)"; + + public static String expectedMainTableBatchIdAndVersionBasedCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + + "(`ID` INT64 NOT NULL,`NAME` STRING NOT NULL,`AMOUNT` FLOAT64,`BIZ_DATE` DATE,`DIGEST` STRING,`VERSION` INT64,`BATCH_ID_IN` INT64 NOT NULL," + + "`BATCH_ID_OUT` INT64,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`) NOT ENFORCED)"; + + public static String expectedBaseTablePlusDigestPlusUpdateTimestampCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`digest` STRING," + + "`batch_update_time` DATETIME NOT NULL," + + "PRIMARY KEY (`id`, `name`, `batch_update_time`) NOT ENFORCED)"; + + public static String expectedBaseTableWithAuditNotPKCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`digest` STRING," + + "`batch_update_time` DATETIME," + + "PRIMARY KEY (`id`, `name`) NOT ENFORCED)"; + + public static String expectedBaseTableWithAuditPKCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`batch_update_time` DATETIME NOT NULL," + + "PRIMARY KEY (`id`, `name`, `batch_update_time`) NOT ENFORCED)"; + + public static String expectedStagingCleanupQuery = "DELETE FROM `mydb`.`staging` as stage WHERE 1 = 1"; + + public static String expectedDropTableQuery = "DROP TABLE IF EXISTS `mydb`.`staging` CASCADE"; + + public static String cleanUpMainTableSql = "DELETE FROM `mydb`.`main` as sink WHERE 1 = 1"; + public static String cleanupMainTableSqlUpperCase = "DELETE FROM `MYDB`.`MAIN` as sink WHERE 1 = 1"; + + public static String expectedMainTableBatchIdBasedCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL,`name` STRING NOT NULL,`amount` FLOAT64,`biz_date` DATE,`digest` STRING," + + "`batch_id_in` INT64 NOT NULL,`batch_id_out` INT64,PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)"; + + public static String expectedMetadataTableCreateQuery = "CREATE TABLE IF NOT EXISTS batch_metadata" + + "(`table_name` STRING(255)," + + "`batch_start_ts_utc` DATETIME," + + "`batch_end_ts_utc` DATETIME," + + "`batch_status` STRING(32)," + + "`table_batch_id` INT64," + + "`staging_filters` JSON)"; + + public static String expectedMetadataTableCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS BATCH_METADATA" + + "(`TABLE_NAME` STRING(255)," + + "`BATCH_START_TS_UTC` DATETIME," + + "`BATCH_END_TS_UTC` DATETIME," + + "`BATCH_STATUS` STRING(32)," + + "`TABLE_BATCH_ID` INT64," + + "`STAGING_FILTERS` JSON)"; + + public static String expectedMainTableBatchIdBasedCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + + "(`ID` INT64 NOT NULL,`NAME` STRING NOT NULL,`AMOUNT` FLOAT64,`BIZ_DATE` DATE,`DIGEST` STRING," + + "`BATCH_ID_IN` INT64 NOT NULL,`BATCH_ID_OUT` INT64,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`) NOT ENFORCED)"; + + public static String expectedMetadataTableIngestQuery = "INSERT INTO batch_metadata (`table_name`, `table_batch_id`, `batch_start_ts_utc`, `batch_end_ts_utc`, `batch_status`)" + + " (SELECT 'main',(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),CURRENT_DATETIME(),'DONE')"; + + public static String expectedMetadataTableIngestWithStagingFiltersQuery = "INSERT INTO batch_metadata " + + "(`table_name`, `table_batch_id`, `batch_start_ts_utc`, `batch_end_ts_utc`, `batch_status`, `staging_filters`) " + + "(SELECT 'main',(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')," + + "CURRENT_DATETIME(),'DONE',PARSE_JSON('{\"batch_id_in\":{\"GT\":5}}'))"; + + public static String expectedMetadataTableIngestQueryWithUpperCase = "INSERT INTO BATCH_METADATA (`TABLE_NAME`, `TABLE_BATCH_ID`, `BATCH_START_TS_UTC`, `BATCH_END_TS_UTC`, `BATCH_STATUS`)" + + " (SELECT 'MAIN',(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN'),PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),CURRENT_DATETIME(),'DONE')"; + + public static String expectedMetadataTableIngestQueryWithPlaceHolders = "INSERT INTO batch_metadata (`table_name`, `table_batch_id`, `batch_start_ts_utc`, `batch_end_ts_utc`, `batch_status`) " + + "(SELECT 'main',{BATCH_ID_PATTERN},PARSE_DATETIME('%Y-%m-%d %H:%M:%S','{BATCH_START_TS_PATTERN}'),'{BATCH_END_TS_PATTERN}','DONE')"; + + public static String expectedMainTableCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`batch_time_in` DATETIME," + + "`batch_time_out` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)"; + + public static String expectedMainTableCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + + "(`ID` INT64 NOT NULL," + + "`NAME` STRING NOT NULL," + + "`AMOUNT` FLOAT64," + + "`BIZ_DATE` DATE," + + "`DIGEST` STRING," + + "`BATCH_ID_IN` INT64 NOT NULL," + + "`BATCH_ID_OUT` INT64," + + "`BATCH_TIME_IN` DATETIME," + + "`BATCH_TIME_OUT` DATETIME," + + "PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`) NOT ENFORCED)"; + + public static String expectedMainTableTimeBasedCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL,`name` STRING NOT NULL,`amount` FLOAT64,`biz_date` DATE,`digest` STRING," + + "`batch_time_in` DATETIME NOT NULL,`batch_time_out` DATETIME,PRIMARY KEY (`id`, `name`, `batch_time_in`) NOT ENFORCED)"; + + public static String expectedMainTableTimeBasedCreateQueryWithUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + + "(`ID` INT64 NOT NULL,`NAME` STRING NOT NULL,`AMOUNT` FLOAT64,`BIZ_DATE` DATE,`DIGEST` STRING," + + "`BATCH_TIME_IN` DATETIME NOT NULL,`BATCH_TIME_OUT` DATETIME,PRIMARY KEY (`ID`, `NAME`, `BATCH_TIME_IN`) NOT ENFORCED)"; + + public static String expectedBitemporalMainTableCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalMainTableWithBatchIdDatetimeCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`batch_time_in` DATETIME," + + "`batch_time_out` DATETIME," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalMainTableWithDatetimeCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_time_in` DATETIME NOT NULL," + + "`batch_time_out` DATETIME," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_time_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyMainTableCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyMainTableBatchIdAndTimeBasedCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`batch_time_in` DATETIME," + + "`batch_time_out` DATETIME," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyMainTableDateTimeBasedCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_time_in` DATETIME NOT NULL," + + "`batch_time_out` DATETIME," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_time_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalMainTableCreateQueryUpperCase = "CREATE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + + "(`ID` INT64 NOT NULL," + + "`NAME` STRING NOT NULL," + + "`AMOUNT` FLOAT64," + + "`DIGEST` STRING," + + "`BATCH_ID_IN` INT64 NOT NULL," + + "`BATCH_ID_OUT` INT64," + + "`VALIDITY_FROM_TARGET` DATETIME NOT NULL," + + "`VALIDITY_THROUGH_TARGET` DATETIME," + + "PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`, `VALIDITY_FROM_TARGET`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyTempTableCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`temp`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyTempTableBatchIdAndTimeBasedCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`temp`(" + + "`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`batch_time_in` DATETIME," + + "`batch_time_out` DATETIME," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyTempTableDateTimeBasedCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`temp`(" + + "`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_time_in` DATETIME NOT NULL," + + "`batch_time_out` DATETIME," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_time_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyStageWithoutDuplicatesTableCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`stagingWithoutDuplicates`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`validity_from_reference` DATETIME NOT NULL," + + "`digest` STRING," + + "PRIMARY KEY (`id`, `name`, `validity_from_reference`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyTempTableWithDeleteIndicatorCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`tempWithDeleteIndicator`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "`delete_indicator` STRING," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyStageWithDataSplitWithoutDuplicatesTableCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`stagingWithoutDuplicates`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`validity_from_reference` DATETIME NOT NULL," + + "`digest` STRING," + + "`data_split` INT64 NOT NULL," + + "PRIMARY KEY (`id`, `name`, `validity_from_reference`, `data_split`) NOT ENFORCED)"; + + public static String expectedBitemporalFromOnlyStageWithDeleteIndicatorWithoutDuplicatesTableCreateQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`stagingWithoutDuplicates`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`validity_from_reference` DATETIME NOT NULL," + + "`digest` STRING," + + "`delete_indicator` STRING," + + "PRIMARY KEY (`id`, `name`, `validity_from_reference`) NOT ENFORCED)"; + +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BitemporalDeltaSourceSpecifiesFromAndThroughTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BitemporalDeltaSourceSpecifiesFromAndThroughTest.java new file mode 100644 index 00000000000..52f3ec8fa93 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BitemporalDeltaSourceSpecifiesFromAndThroughTest.java @@ -0,0 +1,293 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.DataSplitRange; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.testcases.ingestmode.bitemporal.BitemporalDeltaSourceSpecifiesFromAndThroughTestCases; +import org.junit.jupiter.api.Assertions; + +import java.util.List; + +public class BitemporalDeltaSourceSpecifiesFromAndThroughTest extends BitemporalDeltaSourceSpecifiesFromAndThroughTestCases +{ + @Override + public void verifyBitemporalDeltaBatchIdBasedNoDeleteIndNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`validity_from_target` = stage.`validity_from_reference`) " + + "AND (sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `validity_from_target`, `validity_through_target`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`validity_through_reference`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999 " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`validity_from_target` = stage.`validity_from_reference`))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdDateTimeBasedNoDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = " + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1,sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) AND (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE " + + "((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND " + + "((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`validity_from_target` = stage.`validity_from_reference`) AND (sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `validity_from_target`, " + + "`validity_through_target`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`validity_through_reference`," + + "stage.`digest`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage WHERE (NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE " + + "(sink.`batch_id_out` = 999999999) AND (sink.`digest` = stage.`digest`) " + + "AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`validity_from_target` = stage.`validity_from_reference`)))) AND " + + "((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalMainTableWithBatchIdDatetimeCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(1).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= 1) AND (stage.`data_split` <= 1)"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + + verifyStats(operations.get(0), incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedWithDeleteIndNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`validity_from_target` = stage.`validity_from_reference`) " + + "AND ((sink.`digest` <> stage.`digest`) OR (stage.`delete_indicator` IN ('yes','1','true')))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `validity_from_target`, `validity_through_target`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`validity_through_reference`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999 " + + "FROM `mydb`.`staging` as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`validity_from_target` = stage.`validity_from_reference`)))) " + + "AND (stage.`delete_indicator` NOT IN ('yes','1','true')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaDatetimeBasedWithDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND " + + "(stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) " + + "AND (sink.`validity_from_target` = stage.`validity_from_reference`) AND " + + "((sink.`digest` <> stage.`digest`) OR (stage.`delete_indicator` IN ('yes','1','true')))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `validity_from_target`, `validity_through_target`, `digest`, " + + "`batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`," + + "stage.`validity_through_reference`,stage.`digest`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')," + + "'9999-12-31 23:59:59' FROM `mydb`.`staging` as stage WHERE " + + "((NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND " + + "(sink.`name` = stage.`name`)) AND (sink.`validity_from_target` = stage.`validity_from_reference`)))) " + + "AND ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))) AND " + + "(stage.`delete_indicator` NOT IN ('yes','1','true')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalMainTableWithDatetimeCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(1).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= 1) AND (stage.`data_split` <= 1)"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))) as `rowsTerminated`"; + verifyStats(operations.get(0), incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedWithUpperCaseOptimizer(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `MYDB`.`MAIN` as sink " + + "SET sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 " + + "FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1 " + + "WHERE (sink.`BATCH_ID_OUT` = 999999999) AND (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage " + + "WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) " + + "AND (sink.`VALIDITY_FROM_TARGET` = stage.`VALIDITY_FROM_REFERENCE`) " + + "AND (sink.`DIGEST` <> stage.`DIGEST`)))"; + + String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` " + + "(`ID`, `NAME`, `AMOUNT`, `VALIDITY_FROM_TARGET`, `VALIDITY_THROUGH_TARGET`, `DIGEST`, `BATCH_ID_IN`, " + + "`BATCH_ID_OUT`) " + + "(SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`VALIDITY_FROM_REFERENCE`,stage.`VALIDITY_THROUGH_REFERENCE`," + + "stage.`DIGEST`,(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')" + + ",999999999 " + + "FROM `MYDB`.`STAGING` as stage WHERE NOT (EXISTS " + + "(SELECT * FROM `MYDB`.`MAIN` as sink " + + "WHERE (sink.`BATCH_ID_OUT` = 999999999) " + + "AND (sink.`DIGEST` = stage.`DIGEST`) " + + "AND ((sink.`ID` = stage.`ID`) " + + "AND (sink.`NAME` = stage.`NAME`)) " + + "AND (sink.`VALIDITY_FROM_TARGET` = stage.`VALIDITY_FROM_REFERENCE`))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalMainTableCreateQueryUpperCase, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQueryWithUpperCase(), metadataIngestSql.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `INCOMINGRECORDCOUNT` FROM `MYDB`.`STAGING` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `ROWSUPDATED` FROM `MYDB`.`MAIN` as sink WHERE sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `ROWSDELETED`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `MYDB`.`MAIN` as sink WHERE sink.`BATCH_ID_IN` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN'))-(SELECT COUNT(*) FROM `MYDB`.`MAIN` as sink WHERE sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1) as `ROWSINSERTED`"; + String rowsTerminated = "SELECT 0 as `ROWSTERMINATED`"; + + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedWithUserDefinedInfiniteBatchId(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 123456) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`validity_from_target` = stage.`validity_from_reference`) " + + "AND (sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `validity_from_target`, `validity_through_target`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`validity_through_reference`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "123456 " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 123456) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`validity_from_target` = stage.`validity_from_reference`))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + protected String getExpectedMetadataTableIngestQuery() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQuery; + } + + protected String getExpectedMetadataTableIngestQueryWithUpperCase() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithUpperCase; + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BitemporalDeltaSourceSpecifiesFromTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BitemporalDeltaSourceSpecifiesFromTest.java new file mode 100644 index 00000000000..26c900304c2 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/BitemporalDeltaSourceSpecifiesFromTest.java @@ -0,0 +1,1177 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.DataSplitRange; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.testcases.ingestmode.bitemporal.BitemporalDeltaSourceSpecifiesFromTestCases; +import org.junit.jupiter.api.Assertions; + +import java.util.List; + +public class BitemporalDeltaSourceSpecifiesFromTest extends BitemporalDeltaSourceSpecifiesFromTestCases +{ + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + + @Override + public void verifyBitemporalDeltaBatchIdBasedNoDeleteIndNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedStageToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest` FROM `mydb`.`staging` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage " + + "WHERE ((legend_persistence_x.`id` = stage.`id`) AND (legend_persistence_x.`name` = stage.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`temp` as temp " + + "WHERE ((sink.`id` = temp.`id`) AND (sink.`name` = temp.`name`)) AND (sink.`validity_from_target` = temp.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT temp.`id`,temp.`name`,temp.`amount`,temp.`digest`,temp.`batch_id_in`,temp.`batch_id_out`,temp.`validity_from_target`,temp.`validity_through_target` FROM `mydb`.`temp` as temp)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableCreateQuery, preActionsSql.get(2)); + + Assertions.assertEquals(expectedStageToTemp, milestoningSql.get(0)); + Assertions.assertEquals(expectedMainToTemp, milestoningSql.get(1)); + Assertions.assertEquals(expectedUpdateMain, milestoningSql.get(2)); + Assertions.assertEquals(expectedTempToMain, milestoningSql.get(3)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), milestoningSql.get(4)); + + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedNoDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + String expectedStageToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest`,stage.`data_split` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage " + + "WHERE (((legend_persistence_x.`id` = stage.`id`) AND (legend_persistence_x.`name` = stage.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)) AND ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`temp` as temp " + + "WHERE ((sink.`id` = temp.`id`) AND (sink.`name` = temp.`name`)) AND (sink.`validity_from_target` = temp.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT temp.`id`,temp.`name`,temp.`amount`,temp.`digest`,temp.`batch_id_in`,temp.`batch_id_out`,temp.`validity_from_target`,temp.`validity_through_target` FROM `mydb`.`temp` as temp)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableCreateQuery, operations.get(0).preActionsSql().get(2)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedStageToTemp, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTemp, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(expectedUpdateMain, operations.get(0).ingestSql().get(2)); + Assertions.assertEquals(expectedTempToMain, operations.get(0).ingestSql().get(3)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), operations.get(0).ingestSql().get(4)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedStageToTemp, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTemp, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(expectedUpdateMain, operations.get(1).ingestSql().get(2)); + Assertions.assertEquals(expectedTempToMain, operations.get(1).ingestSql().get(3)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), operations.get(1).ingestSql().get(4)); + + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + verifyStats(operations.get(0), enrichSqlWithDataSplits(incomingRecordCount,dataSplitRanges.get(0)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + verifyStats(operations.get(1), enrichSqlWithDataSplits(incomingRecordCount,dataSplitRanges.get(1)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedWithDeleteIndNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedStageToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest`,stage.`delete_indicator` FROM `mydb`.`staging` as stage WHERE stage.`delete_indicator` NOT IN ('yes','1','true')) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage WHERE stage.`delete_indicator` NOT IN ('yes','1','true')) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage WHERE stage.`delete_indicator` NOT IN ('yes','1','true')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage WHERE stage.`delete_indicator` NOT IN ('yes','1','true')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage " + + "WHERE (((legend_persistence_x.`id` = stage.`id`) AND (legend_persistence_x.`name` = stage.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)) AND (stage.`delete_indicator` NOT IN ('yes','1','true'))))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`temp` as temp " + + "WHERE ((sink.`id` = temp.`id`) AND (sink.`name` = temp.`name`)) AND (sink.`validity_from_target` = temp.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT temp.`id`,temp.`name`,temp.`amount`,temp.`digest`,temp.`batch_id_in`,temp.`batch_id_out`,temp.`validity_from_target`,temp.`validity_through_target` FROM `mydb`.`temp` as temp)"; + + String expectedMainToTempForDeletion = "INSERT INTO `mydb`.`tempWithDeleteIndicator` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`, `delete_indicator`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_x.`validity_through_target` as `legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,(CASE WHEN legend_persistence_y.`delete_indicator` IS NULL THEN 0 ELSE 1 END) " + + "FROM " + + "(SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) " + + "AND (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "((sink.`validity_from_target` = stage.`validity_from_reference`) OR (sink.`validity_through_target` = stage.`validity_from_reference`)) " + + "AND (stage.`delete_indicator` IN ('yes','1','true'))))) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT * FROM `mydb`.`staging` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`validity_from_reference`))"; + + String expectedUpdateMainForDeletion = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`tempWithDeleteIndicator` as tempWithDeleteIndicator " + + "WHERE ((sink.`id` = tempWithDeleteIndicator.`id`) AND (sink.`name` = tempWithDeleteIndicator.`name`)) AND (sink.`validity_from_target` = tempWithDeleteIndicator.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMainForDeletion = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`legend_persistence_start_date` as `legend_persistence_start_date`,MAX(legend_persistence_y.`validity_through_target`) as `legend_persistence_end_date`,legend_persistence_x.`batch_id_in`,legend_persistence_x.`batch_id_out` FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`validity_from_target`),'9999-12-31 23:59:59') as `legend_persistence_end_date`,legend_persistence_x.`batch_id_in`,legend_persistence_x.`batch_id_out` " + + "FROM `mydb`.`tempWithDeleteIndicator` as legend_persistence_x " + + "LEFT OUTER JOIN `mydb`.`tempWithDeleteIndicator` as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`validity_from_target` > legend_persistence_x.`validity_from_target`) AND (legend_persistence_y.`delete_indicator` = 0) " + + "WHERE legend_persistence_x.`delete_indicator` = 0 " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`amount`, legend_persistence_x.`digest`, legend_persistence_x.`validity_from_target`, legend_persistence_x.`batch_id_in`, legend_persistence_x.`batch_id_out`) as legend_persistence_x " + + "LEFT OUTER JOIN `mydb`.`tempWithDeleteIndicator` as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`validity_through_target` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`validity_through_target` <= legend_persistence_x.`legend_persistence_end_date`) AND (legend_persistence_y.`delete_indicator` <> 0) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`amount`, legend_persistence_x.`digest`, legend_persistence_x.`legend_persistence_start_date`, legend_persistence_x.`batch_id_in`, legend_persistence_x.`batch_id_out`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableCreateQuery, preActionsSql.get(2)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableWithDeleteIndicatorCreateQuery, preActionsSql.get(3)); + + Assertions.assertEquals(expectedStageToTemp, milestoningSql.get(0)); + Assertions.assertEquals(expectedMainToTemp, milestoningSql.get(1)); + Assertions.assertEquals(expectedUpdateMain, milestoningSql.get(2)); + Assertions.assertEquals(expectedTempToMain, milestoningSql.get(3)); + Assertions.assertEquals(expectedMainToTempForDeletion, milestoningSql.get(4)); + Assertions.assertEquals(expectedUpdateMainForDeletion, milestoningSql.get(5)); + Assertions.assertEquals(expectedTempToMainForDeletion, milestoningSql.get(6)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), milestoningSql.get(7)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`tempWithDeleteIndicator`", "tempWithDeleteIndicator"), milestoningSql.get(8)); + + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedWithDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + String tempName = operations.get(0).preActionsSql().get(2).split("CREATE TABLE IF NOT EXISTS ")[1].split("\\(")[0]; + String tempWithDeleteIndicatorName = operations.get(0).preActionsSql().get(3).split("CREATE TABLE IF NOT EXISTS ")[1].split("\\(")[0]; + + String expectedBitemporalFromOnlyDefaultTempTableCreateQuery = "CREATE TABLE IF NOT EXISTS " + tempName + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + String expectedBitemporalFromOnlyDefaultTempTableWithDeleteIndicatorCreateQuery = "CREATE TABLE IF NOT EXISTS " + tempWithDeleteIndicatorName + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "`delete_indicator` BOOL," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + String expectedStageToTemp = "INSERT INTO " + tempName + " " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest`,stage.`delete_indicator`,stage.`data_split` FROM `mydb`.`staging` as stage WHERE (stage.`delete_indicator` NOT IN ('yes','1','true')) AND ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage WHERE (stage.`delete_indicator` NOT IN ('yes','1','true')) AND ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage WHERE (stage.`delete_indicator` NOT IN ('yes','1','true')) AND ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO " + tempName + " " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage WHERE (stage.`delete_indicator` NOT IN ('yes','1','true')) AND ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage " + + "WHERE ((((legend_persistence_x.`id` = stage.`id`) AND (legend_persistence_x.`name` = stage.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)) AND (stage.`delete_indicator` NOT IN ('yes','1','true'))) AND ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM " + tempName + " as legend_persistence_temp " + + "WHERE ((sink.`id` = legend_persistence_temp.`id`) AND (sink.`name` = legend_persistence_temp.`name`)) AND (sink.`validity_from_target` = legend_persistence_temp.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT legend_persistence_temp.`id`,legend_persistence_temp.`name`,legend_persistence_temp.`amount`,legend_persistence_temp.`digest`,legend_persistence_temp.`batch_id_in`,legend_persistence_temp.`batch_id_out`,legend_persistence_temp.`validity_from_target`,legend_persistence_temp.`validity_through_target` FROM " + tempName + " as legend_persistence_temp)"; + + String expectedMainToTempForDeletion = "INSERT INTO " + tempWithDeleteIndicatorName + " " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`, `delete_indicator`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_x.`validity_through_target` as `legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,(CASE WHEN legend_persistence_y.`delete_indicator` IS NULL THEN 0 ELSE 1 END) " + + "FROM " + + "(SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) " + + "AND (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage " + + "WHERE (((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "((sink.`validity_from_target` = stage.`validity_from_reference`) OR (sink.`validity_through_target` = stage.`validity_from_reference`)) " + + "AND (stage.`delete_indicator` IN ('yes','1','true'))) AND ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))))) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`validity_from_reference`))"; + + String expectedUpdateMainForDeletion = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM " + tempWithDeleteIndicatorName + " as legend_persistence_tempWithDeleteIndicator " + + "WHERE ((sink.`id` = legend_persistence_tempWithDeleteIndicator.`id`) AND (sink.`name` = legend_persistence_tempWithDeleteIndicator.`name`)) AND (sink.`validity_from_target` = legend_persistence_tempWithDeleteIndicator.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMainForDeletion = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`legend_persistence_start_date` as `legend_persistence_start_date`,MAX(legend_persistence_y.`validity_through_target`) as `legend_persistence_end_date`,legend_persistence_x.`batch_id_in`,legend_persistence_x.`batch_id_out` FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`validity_from_target`),'9999-12-31 23:59:59') as `legend_persistence_end_date`,legend_persistence_x.`batch_id_in`,legend_persistence_x.`batch_id_out` " + + "FROM " + tempWithDeleteIndicatorName + " as legend_persistence_x " + + "LEFT OUTER JOIN " + tempWithDeleteIndicatorName + " as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`validity_from_target` > legend_persistence_x.`validity_from_target`) AND (legend_persistence_y.`delete_indicator` = 0) " + + "WHERE legend_persistence_x.`delete_indicator` = 0 " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`amount`, legend_persistence_x.`digest`, legend_persistence_x.`validity_from_target`, legend_persistence_x.`batch_id_in`, legend_persistence_x.`batch_id_out`) as legend_persistence_x " + + "LEFT OUTER JOIN " + tempWithDeleteIndicatorName + " as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`validity_through_target` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`validity_through_target` <= legend_persistence_x.`legend_persistence_end_date`) AND (legend_persistence_y.`delete_indicator` <> 0) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`amount`, legend_persistence_x.`digest`, legend_persistence_x.`legend_persistence_start_date`, legend_persistence_x.`batch_id_in`, legend_persistence_x.`batch_id_out`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + Assertions.assertEquals(expectedBitemporalFromOnlyDefaultTempTableCreateQuery, operations.get(0).preActionsSql().get(2)); + Assertions.assertEquals(expectedBitemporalFromOnlyDefaultTempTableWithDeleteIndicatorCreateQuery, operations.get(0).preActionsSql().get(3)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedStageToTemp, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTemp, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(expectedUpdateMain, operations.get(0).ingestSql().get(2)); + Assertions.assertEquals(expectedTempToMain, operations.get(0).ingestSql().get(3)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTempForDeletion, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(4)); + Assertions.assertEquals(expectedUpdateMainForDeletion, operations.get(0).ingestSql().get(5)); + Assertions.assertEquals(expectedTempToMainForDeletion, operations.get(0).ingestSql().get(6)); + Assertions.assertEquals(getExpectedCleanupSql(tempName, "legend_persistence_temp"), operations.get(0).ingestSql().get(7)); + Assertions.assertEquals(getExpectedCleanupSql(tempWithDeleteIndicatorName, "legend_persistence_tempWithDeleteIndicator"), operations.get(0).ingestSql().get(8)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedStageToTemp, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTemp, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(expectedUpdateMain, operations.get(1).ingestSql().get(2)); + Assertions.assertEquals(expectedTempToMain, operations.get(1).ingestSql().get(3)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTempForDeletion, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(4)); + Assertions.assertEquals(expectedUpdateMainForDeletion, operations.get(1).ingestSql().get(5)); + Assertions.assertEquals(expectedTempToMainForDeletion, operations.get(1).ingestSql().get(6)); + Assertions.assertEquals(getExpectedCleanupSql(tempName, "legend_persistence_temp"), operations.get(1).ingestSql().get(7)); + Assertions.assertEquals(getExpectedCleanupSql(tempWithDeleteIndicatorName, "legend_persistence_tempWithDeleteIndicator"), operations.get(1).ingestSql().get(8)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + verifyStats(operations.get(0), enrichSqlWithDataSplits(incomingRecordCount,dataSplitRanges.get(0)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + verifyStats(operations.get(1), enrichSqlWithDataSplits(incomingRecordCount,dataSplitRanges.get(1)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedNoDeleteIndNoDataSplitsFilterDuplicates(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedStageToStageWithoutDuplicates = "INSERT INTO `mydb`.`stagingWithoutDuplicates` " + + "(`id`, `name`, `amount`, `validity_from_reference`, `digest`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest` FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`digest` = stage.`digest`) AND (sink.`batch_id_out` = 999999999))))"; + + String expectedStageToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest` FROM `mydb`.`stagingWithoutDuplicates` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage " + + "WHERE ((legend_persistence_x.`id` = stage.`id`) AND (legend_persistence_x.`name` = stage.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`temp` as temp " + + "WHERE ((sink.`id` = temp.`id`) AND (sink.`name` = temp.`name`)) AND (sink.`validity_from_target` = temp.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT temp.`id`,temp.`name`,temp.`amount`,temp.`digest`,temp.`batch_id_in`,temp.`batch_id_out`,temp.`validity_from_target`,temp.`validity_through_target` FROM `mydb`.`temp` as temp)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableCreateQuery, preActionsSql.get(2)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyStageWithoutDuplicatesTableCreateQuery, preActionsSql.get(3)); + + Assertions.assertEquals(expectedStageToStageWithoutDuplicates, milestoningSql.get(0)); + Assertions.assertEquals(expectedStageToTemp, milestoningSql.get(1)); + Assertions.assertEquals(expectedMainToTemp, milestoningSql.get(2)); + Assertions.assertEquals(expectedUpdateMain, milestoningSql.get(3)); + Assertions.assertEquals(expectedTempToMain, milestoningSql.get(4)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), milestoningSql.get(5)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`stagingWithoutDuplicates`", "stage"), milestoningSql.get(6)); + + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedNoDeleteIndWithDataSplitsFilterDuplicates(List operations, List dataSplitRanges) + { + String expectedStageToStageWithoutDuplicates = "INSERT INTO `mydb`.`stagingWithoutDuplicates` " + + "(`id`, `name`, `amount`, `validity_from_reference`, `digest`, `data_split`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest`,stage.`data_split` FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`digest` = stage.`digest`) AND (sink.`batch_id_out` = 999999999))))"; + + String expectedStageToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest`,stage.`data_split` FROM `mydb`.`stagingWithoutDuplicates` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage " + + "WHERE (((legend_persistence_x.`id` = stage.`id`) AND (legend_persistence_x.`name` = stage.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)) AND ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`temp` as temp " + + "WHERE ((sink.`id` = temp.`id`) AND (sink.`name` = temp.`name`)) AND (sink.`validity_from_target` = temp.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT temp.`id`,temp.`name`,temp.`amount`,temp.`digest`,temp.`batch_id_in`,temp.`batch_id_out`,temp.`validity_from_target`,temp.`validity_through_target` FROM `mydb`.`temp` as temp)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableCreateQuery, operations.get(0).preActionsSql().get(2)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyStageWithDataSplitWithoutDuplicatesTableCreateQuery, operations.get(0).preActionsSql().get(3)); + + Assertions.assertEquals(expectedStageToStageWithoutDuplicates, operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedStageToTemp, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTemp, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(2)); + Assertions.assertEquals(expectedUpdateMain, operations.get(0).ingestSql().get(3)); + Assertions.assertEquals(expectedTempToMain, operations.get(0).ingestSql().get(4)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), operations.get(0).ingestSql().get(5)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`stagingWithoutDuplicates`", "stage"), operations.get(0).ingestSql().get(6)); + + Assertions.assertEquals(expectedStageToStageWithoutDuplicates, operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedStageToTemp, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTemp, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(2)); + Assertions.assertEquals(expectedUpdateMain, operations.get(1).ingestSql().get(3)); + Assertions.assertEquals(expectedTempToMain, operations.get(1).ingestSql().get(4)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), operations.get(1).ingestSql().get(5)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`stagingWithoutDuplicates`", "stage"), operations.get(1).ingestSql().get(6)); + + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + + Assertions.assertEquals(2, operations.size()); + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + verifyStats(operations.get(0), enrichSqlWithDataSplits(incomingRecordCount,dataSplitRanges.get(0)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + verifyStats(operations.get(1), enrichSqlWithDataSplits(incomingRecordCount,dataSplitRanges.get(1)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedWithDeleteIndNoDataSplitsFilterDuplicates(GeneratorResult operations) + { + + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedStageToStageWithoutDuplicates = "INSERT INTO `mydb`.`stagingWithoutDuplicates` " + + "(`id`, `name`, `amount`, `validity_from_reference`, `digest`, `delete_indicator`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest`,stage.`delete_indicator` FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`digest` = stage.`digest`) AND (sink.`batch_id_out` = 999999999))))"; + + String expectedStageToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest`,stage.`delete_indicator` FROM `mydb`.`stagingWithoutDuplicates` as stage WHERE stage.`delete_indicator` NOT IN ('yes','1','true')) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage WHERE stage.`delete_indicator` NOT IN ('yes','1','true')) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage WHERE stage.`delete_indicator` NOT IN ('yes','1','true')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage WHERE stage.`delete_indicator` NOT IN ('yes','1','true')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`stagingWithoutDuplicates` as stage " + + "WHERE (((legend_persistence_x.`id` = stage.`id`) AND (legend_persistence_x.`name` = stage.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)) AND (stage.`delete_indicator` NOT IN ('yes','1','true'))))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`temp` as temp " + + "WHERE ((sink.`id` = temp.`id`) AND (sink.`name` = temp.`name`)) AND (sink.`validity_from_target` = temp.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT temp.`id`,temp.`name`,temp.`amount`,temp.`digest`,temp.`batch_id_in`,temp.`batch_id_out`,temp.`validity_from_target`,temp.`validity_through_target` FROM `mydb`.`temp` as temp)"; + + String expectedMainToTempForDeletion = "INSERT INTO `mydb`.`tempWithDeleteIndicator` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`, `delete_indicator`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_x.`validity_through_target` as `legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,(CASE WHEN legend_persistence_y.`delete_indicator` IS NULL THEN 0 ELSE 1 END) " + + "FROM " + + "(SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) " + + "AND (EXISTS " + + "(SELECT * FROM `mydb`.`stagingWithoutDuplicates` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "((sink.`validity_from_target` = stage.`validity_from_reference`) OR (sink.`validity_through_target` = stage.`validity_from_reference`)) " + + "AND (stage.`delete_indicator` IN ('yes','1','true'))))) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT * FROM `mydb`.`stagingWithoutDuplicates` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`validity_from_reference`))"; + + String expectedUpdateMainForDeletion = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`tempWithDeleteIndicator` as tempWithDeleteIndicator " + + "WHERE ((sink.`id` = tempWithDeleteIndicator.`id`) AND (sink.`name` = tempWithDeleteIndicator.`name`)) AND (sink.`validity_from_target` = tempWithDeleteIndicator.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMainForDeletion = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`legend_persistence_start_date` as `legend_persistence_start_date`,MAX(legend_persistence_y.`validity_through_target`) as `legend_persistence_end_date`,legend_persistence_x.`batch_id_in`,legend_persistence_x.`batch_id_out` FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`validity_from_target`),'9999-12-31 23:59:59') as `legend_persistence_end_date`,legend_persistence_x.`batch_id_in`,legend_persistence_x.`batch_id_out` " + + "FROM `mydb`.`tempWithDeleteIndicator` as legend_persistence_x " + + "LEFT OUTER JOIN `mydb`.`tempWithDeleteIndicator` as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`validity_from_target` > legend_persistence_x.`validity_from_target`) AND (legend_persistence_y.`delete_indicator` = 0) " + + "WHERE legend_persistence_x.`delete_indicator` = 0 " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`amount`, legend_persistence_x.`digest`, legend_persistence_x.`validity_from_target`, legend_persistence_x.`batch_id_in`, legend_persistence_x.`batch_id_out`) as legend_persistence_x " + + "LEFT OUTER JOIN `mydb`.`tempWithDeleteIndicator` as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`validity_through_target` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`validity_through_target` <= legend_persistence_x.`legend_persistence_end_date`) AND (legend_persistence_y.`delete_indicator` <> 0) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`amount`, legend_persistence_x.`digest`, legend_persistence_x.`legend_persistence_start_date`, legend_persistence_x.`batch_id_in`, legend_persistence_x.`batch_id_out`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableCreateQuery, preActionsSql.get(2)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableWithDeleteIndicatorCreateQuery, preActionsSql.get(3)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyStageWithDeleteIndicatorWithoutDuplicatesTableCreateQuery, preActionsSql.get(4)); + + Assertions.assertEquals(expectedStageToStageWithoutDuplicates, milestoningSql.get(0)); + Assertions.assertEquals(expectedStageToTemp, milestoningSql.get(1)); + Assertions.assertEquals(expectedMainToTemp, milestoningSql.get(2)); + Assertions.assertEquals(expectedUpdateMain, milestoningSql.get(3)); + Assertions.assertEquals(expectedTempToMain, milestoningSql.get(4)); + Assertions.assertEquals(expectedMainToTempForDeletion, milestoningSql.get(5)); + Assertions.assertEquals(expectedUpdateMainForDeletion, milestoningSql.get(6)); + Assertions.assertEquals(expectedTempToMainForDeletion, milestoningSql.get(7)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), milestoningSql.get(8)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`tempWithDeleteIndicator`", "tempWithDeleteIndicator"), milestoningSql.get(9)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`stagingWithoutDuplicates`", "stage"), milestoningSql.get(10)); + + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedWithDeleteIndWithDataSplitsFilterDuplicates(List operations, List dataSplitRanges) + { + String tempName = operations.get(0).preActionsSql().get(2).split("CREATE TABLE IF NOT EXISTS ")[1].split("\\(")[0]; + String tempWithDeleteIndicatorName = operations.get(0).preActionsSql().get(3).split("CREATE TABLE IF NOT EXISTS ")[1].split("\\(")[0]; + String stageWithoutDuplicatesName = operations.get(0).preActionsSql().get(4).split("CREATE TABLE IF NOT EXISTS ")[1].split("\\(")[0]; + + String expectedBitemporalFromOnlyDefaultTempTableCreateQuery = "CREATE TABLE IF NOT EXISTS " + tempName + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + String expectedBitemporalFromOnlyDefaultTempTableWithDeleteIndicatorCreateQuery = "CREATE TABLE IF NOT EXISTS " + tempWithDeleteIndicatorName + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`validity_from_target` DATETIME NOT NULL," + + "`validity_through_target` DATETIME," + + "`delete_indicator` BOOL," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `validity_from_target`) NOT ENFORCED)"; + + String expectedBitemporalFromOnlyStageWithDeleteIndicatorWithDataSplitWithoutDuplicatesTableCreateQuery = "CREATE TABLE IF NOT EXISTS " + stageWithoutDuplicatesName + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`validity_from_reference` DATETIME NOT NULL," + + "`digest` STRING," + + "`delete_indicator` STRING," + + "`data_split` INT64 NOT NULL," + + "PRIMARY KEY (`id`, `name`, `validity_from_reference`, `data_split`) NOT ENFORCED)"; + + String expectedStageToStageWithoutDuplicates = "INSERT INTO " + stageWithoutDuplicatesName + " " + + "(`id`, `name`, `amount`, `validity_from_reference`, `digest`, `delete_indicator`, `data_split`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest`,stage.`delete_indicator`,stage.`data_split` FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`digest` = stage.`digest`) AND (sink.`batch_id_out` = 999999999))))"; + + String expectedStageToTemp = "INSERT INTO " + tempName + " " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT legend_persistence_stageWithoutDuplicates.`id`,legend_persistence_stageWithoutDuplicates.`name`,legend_persistence_stageWithoutDuplicates.`amount`,legend_persistence_stageWithoutDuplicates.`validity_from_reference`,legend_persistence_stageWithoutDuplicates.`digest`,legend_persistence_stageWithoutDuplicates.`delete_indicator`,legend_persistence_stageWithoutDuplicates.`data_split` FROM " + stageWithoutDuplicatesName + " as legend_persistence_stageWithoutDuplicates WHERE (legend_persistence_stageWithoutDuplicates.`delete_indicator` NOT IN ('yes','1','true')) AND ((legend_persistence_stageWithoutDuplicates.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (legend_persistence_stageWithoutDuplicates.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM " + stageWithoutDuplicatesName + " as legend_persistence_stageWithoutDuplicates WHERE (legend_persistence_stageWithoutDuplicates.`delete_indicator` NOT IN ('yes','1','true')) AND ((legend_persistence_stageWithoutDuplicates.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (legend_persistence_stageWithoutDuplicates.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM " + stageWithoutDuplicatesName + " as legend_persistence_stageWithoutDuplicates WHERE (legend_persistence_stageWithoutDuplicates.`delete_indicator` NOT IN ('yes','1','true')) AND ((legend_persistence_stageWithoutDuplicates.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (legend_persistence_stageWithoutDuplicates.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO " + tempName + " " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM " + stageWithoutDuplicatesName + " as legend_persistence_stageWithoutDuplicates WHERE (legend_persistence_stageWithoutDuplicates.`delete_indicator` NOT IN ('yes','1','true')) AND ((legend_persistence_stageWithoutDuplicates.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (legend_persistence_stageWithoutDuplicates.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM " + stageWithoutDuplicatesName + " as legend_persistence_stageWithoutDuplicates " + + "WHERE ((((legend_persistence_x.`id` = legend_persistence_stageWithoutDuplicates.`id`) AND (legend_persistence_x.`name` = legend_persistence_stageWithoutDuplicates.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = legend_persistence_stageWithoutDuplicates.`validity_from_reference`)) AND (legend_persistence_stageWithoutDuplicates.`delete_indicator` NOT IN ('yes','1','true'))) AND ((legend_persistence_stageWithoutDuplicates.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (legend_persistence_stageWithoutDuplicates.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM " + tempName + " as legend_persistence_temp " + + "WHERE ((sink.`id` = legend_persistence_temp.`id`) AND (sink.`name` = legend_persistence_temp.`name`)) AND (sink.`validity_from_target` = legend_persistence_temp.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT legend_persistence_temp.`id`,legend_persistence_temp.`name`,legend_persistence_temp.`amount`,legend_persistence_temp.`digest`,legend_persistence_temp.`batch_id_in`,legend_persistence_temp.`batch_id_out`,legend_persistence_temp.`validity_from_target`,legend_persistence_temp.`validity_through_target` FROM " + tempName + " as legend_persistence_temp)"; + + String expectedMainToTempForDeletion = "INSERT INTO " + tempWithDeleteIndicatorName + " " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`, `delete_indicator`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_x.`validity_through_target` as `legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,(CASE WHEN legend_persistence_y.`delete_indicator` IS NULL THEN 0 ELSE 1 END) " + + "FROM " + + "(SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) " + + "AND (EXISTS " + + "(SELECT * FROM " + stageWithoutDuplicatesName + " as legend_persistence_stageWithoutDuplicates " + + "WHERE (((sink.`id` = legend_persistence_stageWithoutDuplicates.`id`) AND (sink.`name` = legend_persistence_stageWithoutDuplicates.`name`)) AND " + + "((sink.`validity_from_target` = legend_persistence_stageWithoutDuplicates.`validity_from_reference`) OR (sink.`validity_through_target` = legend_persistence_stageWithoutDuplicates.`validity_from_reference`)) " + + "AND (legend_persistence_stageWithoutDuplicates.`delete_indicator` IN ('yes','1','true'))) AND ((legend_persistence_stageWithoutDuplicates.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (legend_persistence_stageWithoutDuplicates.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}'))))) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT * FROM " + stageWithoutDuplicatesName + " as legend_persistence_stageWithoutDuplicates WHERE (legend_persistence_stageWithoutDuplicates.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (legend_persistence_stageWithoutDuplicates.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`validity_from_reference`))"; + + String expectedUpdateMainForDeletion = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM " + tempWithDeleteIndicatorName + " as legend_persistence_tempWithDeleteIndicator " + + "WHERE ((sink.`id` = legend_persistence_tempWithDeleteIndicator.`id`) AND (sink.`name` = legend_persistence_tempWithDeleteIndicator.`name`)) AND (sink.`validity_from_target` = legend_persistence_tempWithDeleteIndicator.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMainForDeletion = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`legend_persistence_start_date` as `legend_persistence_start_date`,MAX(legend_persistence_y.`validity_through_target`) as `legend_persistence_end_date`,legend_persistence_x.`batch_id_in`,legend_persistence_x.`batch_id_out` FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`validity_from_target`),'9999-12-31 23:59:59') as `legend_persistence_end_date`,legend_persistence_x.`batch_id_in`,legend_persistence_x.`batch_id_out` " + + "FROM " + tempWithDeleteIndicatorName + " as legend_persistence_x " + + "LEFT OUTER JOIN " + tempWithDeleteIndicatorName + " as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`validity_from_target` > legend_persistence_x.`validity_from_target`) AND (legend_persistence_y.`delete_indicator` = 0) " + + "WHERE legend_persistence_x.`delete_indicator` = 0 " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`amount`, legend_persistence_x.`digest`, legend_persistence_x.`validity_from_target`, legend_persistence_x.`batch_id_in`, legend_persistence_x.`batch_id_out`) as legend_persistence_x " + + "LEFT OUTER JOIN " + tempWithDeleteIndicatorName + " as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`validity_through_target` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`validity_through_target` <= legend_persistence_x.`legend_persistence_end_date`) AND (legend_persistence_y.`delete_indicator` <> 0) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`amount`, legend_persistence_x.`digest`, legend_persistence_x.`legend_persistence_start_date`, legend_persistence_x.`batch_id_in`, legend_persistence_x.`batch_id_out`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + Assertions.assertEquals(expectedBitemporalFromOnlyDefaultTempTableCreateQuery, operations.get(0).preActionsSql().get(2)); + Assertions.assertEquals(expectedBitemporalFromOnlyDefaultTempTableWithDeleteIndicatorCreateQuery, operations.get(0).preActionsSql().get(3)); + Assertions.assertEquals(expectedBitemporalFromOnlyStageWithDeleteIndicatorWithDataSplitWithoutDuplicatesTableCreateQuery, operations.get(0).preActionsSql().get(4)); + + Assertions.assertEquals(expectedStageToStageWithoutDuplicates, operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedStageToTemp, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTemp, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(2)); + Assertions.assertEquals(expectedUpdateMain, operations.get(0).ingestSql().get(3)); + Assertions.assertEquals(expectedTempToMain, operations.get(0).ingestSql().get(4)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTempForDeletion, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(5)); + Assertions.assertEquals(expectedUpdateMainForDeletion, operations.get(0).ingestSql().get(6)); + Assertions.assertEquals(expectedTempToMainForDeletion, operations.get(0).ingestSql().get(7)); + Assertions.assertEquals(getExpectedCleanupSql(tempName, "legend_persistence_temp"), operations.get(0).ingestSql().get(8)); + Assertions.assertEquals(getExpectedCleanupSql(tempWithDeleteIndicatorName, "legend_persistence_tempWithDeleteIndicator"), operations.get(0).ingestSql().get(9)); + Assertions.assertEquals(getExpectedCleanupSql(stageWithoutDuplicatesName, "legend_persistence_stageWithoutDuplicates"), operations.get(0).ingestSql().get(10)); + + Assertions.assertEquals(expectedStageToStageWithoutDuplicates, operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedStageToTemp, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTemp, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(2)); + Assertions.assertEquals(expectedUpdateMain, operations.get(1).ingestSql().get(3)); + Assertions.assertEquals(expectedTempToMain, operations.get(1).ingestSql().get(4)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMainToTempForDeletion, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(5)); + Assertions.assertEquals(expectedUpdateMainForDeletion, operations.get(1).ingestSql().get(6)); + Assertions.assertEquals(expectedTempToMainForDeletion, operations.get(1).ingestSql().get(7)); + Assertions.assertEquals(getExpectedCleanupSql(tempName, "legend_persistence_temp"), operations.get(1).ingestSql().get(8)); + Assertions.assertEquals(getExpectedCleanupSql(tempWithDeleteIndicatorName, "legend_persistence_tempWithDeleteIndicator"), operations.get(1).ingestSql().get(9)); + Assertions.assertEquals(getExpectedCleanupSql(stageWithoutDuplicatesName, "legend_persistence_stageWithoutDuplicates"), operations.get(1).ingestSql().get(10)); + + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + + Assertions.assertEquals(2, operations.size()); + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`) AND (sink2.`validity_from_target` = sink.`validity_from_target`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + verifyStats(operations.get(0), enrichSqlWithDataSplits(incomingRecordCount,dataSplitRanges.get(0)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + verifyStats(operations.get(1), enrichSqlWithDataSplits(incomingRecordCount,dataSplitRanges.get(1)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaBatchIdBasedWithPlaceholders(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedStageToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,{BATCH_ID_PATTERN},999999999 " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest` FROM `mydb`.`staging` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`,{BATCH_ID_PATTERN},999999999 " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage " + + "WHERE ((legend_persistence_x.`id` = stage.`id`) AND (legend_persistence_x.`name` = stage.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = {BATCH_ID_PATTERN}-1 " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`temp` as temp " + + "WHERE ((sink.`id` = temp.`id`) AND (sink.`name` = temp.`name`)) AND (sink.`validity_from_target` = temp.`validity_from_target`))) " + + "AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT temp.`id`,temp.`name`,temp.`amount`,temp.`digest`,temp.`batch_id_in`,temp.`batch_id_out`,temp.`validity_from_target`,temp.`validity_through_target` FROM `mydb`.`temp` as temp)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableCreateQuery, preActionsSql.get(2)); + + Assertions.assertEquals(expectedStageToTemp, milestoningSql.get(0)); + Assertions.assertEquals(expectedMainToTemp, milestoningSql.get(1)); + Assertions.assertEquals(expectedUpdateMain, milestoningSql.get(2)); + Assertions.assertEquals(expectedTempToMain, milestoningSql.get(3)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithPlaceHolders, metadataIngestSql.get(0)); + } + + @Override + public void verifyBitemporalDeltaBatchIdAndTimeBasedNoDeleteIndNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedStageToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`," + + "legend_persistence_y.`legend_persistence_end_date`,(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest` FROM `mydb`.`staging` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM " + + "(SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_id_in`,sink.`batch_id_out`,sink.`batch_time_in`," + + "sink.`batch_time_out`,sink.`validity_from_target`,sink.`validity_through_target` FROM `mydb`.`main` as sink " + + "WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` " + + "FROM (SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM (SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` FROM `mydb`.`main` as sink " + + "WHERE sink.`batch_id_out` = 999999999) as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) " + + "AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) " + + "AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS (SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage " + + "WHERE ((legend_persistence_x.`id` = stage.`id`) AND (legend_persistence_x.`name` = stage.`name`)) " + + "AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) " + + "AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (EXISTS " + + "(SELECT * FROM `mydb`.`temp` as temp WHERE ((sink.`id` = temp.`id`) AND (sink.`name` = temp.`name`)) " + + "AND (sink.`validity_from_target` = temp.`validity_from_target`))) AND (sink.`batch_id_out` = 999999999)"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT temp.`id`,temp.`name`,temp.`amount`,temp.`digest`,temp.`batch_id_in`,temp.`batch_id_out`," + + "temp.`batch_time_in`,temp.`batch_time_out`,temp.`validity_from_target`,temp.`validity_through_target` " + + "FROM `mydb`.`temp` as temp)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableBatchIdAndTimeBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableBatchIdAndTimeBasedCreateQuery, preActionsSql.get(2)); + + Assertions.assertEquals(expectedStageToTemp, milestoningSql.get(0)); + Assertions.assertEquals(expectedMainToTemp, milestoningSql.get(1)); + Assertions.assertEquals(expectedUpdateMain, milestoningSql.get(2)); + Assertions.assertEquals(expectedTempToMain, milestoningSql.get(3)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), milestoningSql.get(4)); + + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyBitemporalDeltaDateTimeBasedNoDeleteIndNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedStageToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_time_in`, `batch_time_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`,legend_persistence_x.`validity_from_reference` as `legend_persistence_start_date`," + + "legend_persistence_y.`legend_persistence_end_date`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`validity_from_reference`,stage.`digest` FROM `mydb`.`staging` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),MIN(legend_persistence_x.`legend_persistence_end_date`)) as `legend_persistence_end_date` " + + "FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`,COALESCE(MIN(legend_persistence_y.`legend_persistence_start_date`),'9999-12-31 23:59:59') as `legend_persistence_end_date` " + + "FROM " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date` FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = '9999-12-31 23:59:59') as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` < legend_persistence_y.`legend_persistence_start_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "LEFT OUTER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) AND (legend_persistence_x.`validity_from_reference` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedMainToTemp = "INSERT INTO `mydb`.`temp` " + + "(`id`, `name`, `amount`, `digest`, `validity_from_target`, `validity_through_target`, `batch_time_in`, `batch_time_out`) " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`amount`,legend_persistence_x.`digest`," + + "legend_persistence_x.`validity_from_target` as `legend_persistence_start_date`,legend_persistence_y.`legend_persistence_end_date`," + + "PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM (SELECT sink.`id`,sink.`name`,sink.`amount`,sink.`digest`,sink.`batch_time_in`," + + "sink.`batch_time_out`,sink.`validity_from_target`,sink.`validity_through_target` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = '9999-12-31 23:59:59') as legend_persistence_x " + + "INNER JOIN " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`," + + "legend_persistence_x.`legend_persistence_end_date` as `legend_persistence_end_date` FROM " + + "(SELECT legend_persistence_x.`id`,legend_persistence_x.`name`,legend_persistence_x.`legend_persistence_start_date`," + + "MIN(legend_persistence_y.`legend_persistence_start_date`) as `legend_persistence_end_date` " + + "FROM (SELECT `id`,`name`,`validity_from_target` as `legend_persistence_start_date`,`validity_through_target` as `legend_persistence_end_date` " + + "FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = '9999-12-31 23:59:59') as legend_persistence_x " + + "INNER JOIN " + + "(SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` FROM `mydb`.`staging` as stage) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) " + + "AND (legend_persistence_y.`legend_persistence_start_date` > legend_persistence_x.`legend_persistence_start_date`) " + + "AND (legend_persistence_y.`legend_persistence_start_date` < legend_persistence_x.`legend_persistence_end_date`) " + + "GROUP BY legend_persistence_x.`id`, legend_persistence_x.`name`, legend_persistence_x.`legend_persistence_start_date`) as legend_persistence_x " + + "WHERE NOT (EXISTS (SELECT `id`,`name`,`validity_from_reference` as `legend_persistence_start_date` " + + "FROM `mydb`.`staging` as stage WHERE ((legend_persistence_x.`id` = stage.`id`) AND " + + "(legend_persistence_x.`name` = stage.`name`)) AND (legend_persistence_x.`legend_persistence_start_date` = stage.`validity_from_reference`)))) as legend_persistence_y " + + "ON ((legend_persistence_x.`id` = legend_persistence_y.`id`) AND (legend_persistence_x.`name` = legend_persistence_y.`name`)) " + + "AND (legend_persistence_x.`validity_from_target` = legend_persistence_y.`legend_persistence_start_date`))"; + + String expectedUpdateMain = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (EXISTS (SELECT * FROM `mydb`.`temp` as temp WHERE " + + "((sink.`id` = temp.`id`) AND (sink.`name` = temp.`name`)) AND " + + "(sink.`validity_from_target` = temp.`validity_from_target`))) AND (sink.`batch_time_out` = '9999-12-31 23:59:59')"; + + String expectedTempToMain = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_time_in`, `batch_time_out`, `validity_from_target`, `validity_through_target`) " + + "(SELECT temp.`id`,temp.`name`,temp.`amount`,temp.`digest`,temp.`batch_time_in`,temp.`batch_time_out`,temp.`validity_from_target`,temp.`validity_through_target` " + + "FROM `mydb`.`temp` as temp)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyMainTableDateTimeBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedBitemporalFromOnlyTempTableDateTimeBasedCreateQuery, preActionsSql.get(2)); + + Assertions.assertEquals(expectedStageToTemp, milestoningSql.get(0)); + Assertions.assertEquals(expectedMainToTemp, milestoningSql.get(1)); + Assertions.assertEquals(expectedUpdateMain, milestoningSql.get(2)); + Assertions.assertEquals(expectedTempToMain, milestoningSql.get(3)); + Assertions.assertEquals(getExpectedCleanupSql("`mydb`.`temp`", "temp"), milestoningSql.get(4)); + + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) as `rowsInserted`"; + + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + protected String getExpectedMetadataTableIngestQuery() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQuery; + } + + protected String getExpectedCleanupSql(String fullName, String alias) + { + return String.format("DELETE FROM %s as %s WHERE 1 = 1", fullName, alias); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeTest.java new file mode 100644 index 00000000000..1e8dbb810ae --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeTest.java @@ -0,0 +1,222 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; + +import java.time.Clock; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class IngestModeTest +{ + protected final ZonedDateTime fixedExecutionZonedDateTime = ZonedDateTime.of(2000, 1, 1, 0, 0, 0, 0, ZoneOffset.UTC); + protected final Clock fixedClock_2000_01_01 = Clock.fixed(fixedExecutionZonedDateTime.toInstant(), ZoneOffset.UTC); + + String mainDbName = "mydb"; + String mainTableName = "main"; + String mainTableAlias = "sink"; + + String stagingDbName = "mydb"; + String stagingTableName = "staging"; + String stagingTableAlias = "stage"; + + String digestField = "digest"; + String batchIdInField = "batch_id_in"; + String batchIdOutField = "batch_id_out"; + String batchTimeInField = "batch_time_in"; + String batchTimeOutField = "batch_time_out"; + + String[] partitionKeys = new String[] {"biz_date"}; + HashMap> partitionFilter = new HashMap>() + {{ + put("biz_date", new HashSet<>(Arrays.asList("2000-01-01 00:00:00", "2000-01-02 00:00:00"))); + }}; + + // Base Columns: Primary keys : id, name + Field id = Field.builder().name("id").type(FieldType.of(DataType.INT, Optional.empty(), Optional.empty())).primaryKey(true).build(); + Field name = Field.builder().name("name").type(FieldType.of(DataType.VARCHAR, Optional.empty(), Optional.empty())).primaryKey(true).build(); + Field amount = Field.builder().name("amount").type(FieldType.of(DataType.DOUBLE, Optional.empty(), Optional.empty())).build(); + Field bizDate = Field.builder().name("biz_date").type(FieldType.of(DataType.DATE, Optional.empty(), Optional.empty())).build(); + + // Problematic Columns + Field batchIdInNonPrimary = Field.builder().name(batchIdInField).type(FieldType.of(DataType.INT, Optional.empty(), Optional.empty())).build(); + Field batchTimeInNonPrimary = Field.builder().name(batchTimeInField).type(FieldType.of(DataType.DATETIME, Optional.empty(), Optional.empty())).build(); + + // Milestoning Columns + Field digest = Field.builder().name(digestField).type(FieldType.of(DataType.VARCHAR, Optional.empty(), Optional.empty())).build(); + Field batchIdIn = Field.builder().name(batchIdInField).type(FieldType.of(DataType.INT, Optional.empty(), Optional.empty())).primaryKey(true).build(); + Field batchIdOut = Field.builder().name(batchIdOutField).type(FieldType.of(DataType.INT, Optional.empty(), Optional.empty())).build(); + Field batchTimeIn = Field.builder().name(batchTimeInField).type(FieldType.of(DataType.DATETIME, Optional.empty(), Optional.empty())).primaryKey(true).build(); + Field batchTimeOut = Field.builder().name(batchTimeOutField).type(FieldType.of(DataType.DATETIME, Optional.empty(), Optional.empty())).build(); + + SchemaDefinition mainTableSchema = SchemaDefinition.builder() + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchIdIn) + .addFields(batchIdOut) + .addFields(batchTimeIn) + .addFields(batchTimeOut) + .build(); + + SchemaDefinition mainTableSchemaWithBatchIdInNotPrimary = SchemaDefinition.builder() + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchIdInNonPrimary) + .addFields(batchIdOut) + .addFields(batchTimeIn) + .addFields(batchTimeOut) + .build(); + + SchemaDefinition mainTableBatchIdBasedSchema = SchemaDefinition.builder() + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchIdIn) + .addFields(batchIdOut) + .build(); + + SchemaDefinition mainTableBatchIdBasedSchemaWithBatchIdInNotPrimary = SchemaDefinition.builder() + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchIdInNonPrimary) + .addFields(batchIdOut) + .build(); + + SchemaDefinition mainTableTimeBasedSchema = SchemaDefinition.builder() + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchTimeIn) + .addFields(batchTimeOut) + .build(); + + SchemaDefinition mainTableTimeBasedSchemaWithBatchTimeInNotPrimary = SchemaDefinition.builder() + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchTimeInNonPrimary) + .addFields(batchTimeOut) + .build(); + + SchemaDefinition baseTableSchemaWithDigest = SchemaDefinition.builder() + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .build(); + + SchemaDefinition stagingTableSchemaWithLimitedColumns = SchemaDefinition.builder() + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(digest) + .build(); + + String expectedMetadataTableCreateQuery = "CREATE REFERENCE TABLE IF NOT EXISTS batch_metadata" + + "(`table_name` VARCHAR(255)," + + "`batch_start_ts_utc` DATETIME," + + "`batch_end_ts_utc` DATETIME," + + "`batch_status` VARCHAR(32)," + + "`table_batch_id` INTEGER)"; + + String expectedMetadataTableCreateQueryWithUpperCase = "CREATE REFERENCE TABLE IF NOT EXISTS BATCH_METADATA" + + "(`TABLE_NAME` VARCHAR(255)," + + "`BATCH_START_TS_UTC` DATETIME," + + "`BATCH_END_TS_UTC` DATETIME," + + "`BATCH_STATUS` VARCHAR(32)," + + "`TABLE_BATCH_ID` INTEGER)"; + + protected String expectedMetadataTableIngestQuery = "INSERT INTO batch_metadata (`table_name`, `table_batch_id`, `batch_start_ts_utc`, `batch_end_ts_utc`, `batch_status`) (SELECT 'main',(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),CURRENT_TIMESTAMP(),'DONE')"; + + protected String expectedMetadataTableIngestQueryWithUpperCase = "INSERT INTO BATCH_METADATA (`TABLE_NAME`, `TABLE_BATCH_ID`, `BATCH_START_TS_UTC`, `BATCH_END_TS_UTC`, `BATCH_STATUS`) (SELECT 'main',(SELECT COALESCE(MAX(batch_metadata.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as batch_metadata WHERE batch_metadata.`TABLE_NAME` = 'main'),PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),CURRENT_TIMESTAMP(),'DONE')"; + + String expectedStagingCleanupQuery = "DELETE FROM `mydb`.`staging` as stage"; + + String expectedMainTableCreateQuery = "CREATE REFERENCE TABLE IF NOT EXISTS `mydb`.`main`" + + "(`id` INTEGER," + + "`name` VARCHAR(256)," + + "`amount` DOUBLE," + + "`biz_date` DATE," + + "`digest` VARCHAR(256)," + + "`batch_id_in` INTEGER," + + "`batch_id_out` INTEGER," + + "`batch_time_in` DATETIME," + + "`batch_time_out` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `batch_time_in`))"; + + String expectedMainTableCreateQueryWithUpperCase = "CREATE REFERENCE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + + "(`ID` INTEGER," + + "`NAME` VARCHAR(256)," + + "`AMOUNT` DOUBLE," + + "`BIZ_DATE` DATE," + + "`DIGEST` VARCHAR(256)," + + "`BATCH_ID_IN` INTEGER," + + "`BATCH_ID_OUT` INTEGER," + + "`BATCH_TIME_IN` DATETIME," + + "`BATCH_TIME_OUT` DATETIME," + + "PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`, `BATCH_TIME_IN`))"; + + String expectedMainTableBatchIdBasedCreateQuery = "CREATE REFERENCE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INTEGER,`name` VARCHAR(256),`amount` DOUBLE,`biz_date` DATE,`digest` VARCHAR(256)," + + "`batch_id_in` INTEGER,`batch_id_out` INTEGER,PRIMARY KEY (`id`, `name`, `batch_id_in`))"; + + String expectedMainTableBatchIdBasedCreateQueryWithUpperCase = "CREATE REFERENCE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + + "(`ID` INTEGER,`NAME` VARCHAR(256),`AMOUNT` DOUBLE,`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," + + "`BATCH_ID_IN` INTEGER,`BATCH_ID_OUT` INTEGER,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`))"; + + String expectedMainTableTimeBasedCreateQuery = "CREATE REFERENCE TABLE IF NOT EXISTS `mydb`.`main`(" + + "`id` INTEGER,`name` VARCHAR(256),`amount` DOUBLE,`biz_date` DATE,`digest` VARCHAR(256)," + + "`batch_time_in` DATETIME,`batch_time_out` DATETIME,PRIMARY KEY (`id`, `name`, `batch_time_in`))"; + + String expectedMainTableTimeBasedCreateQueryWithUpperCase = "CREATE REFERENCE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + + "(`ID` INTEGER,`NAME` VARCHAR(256),`AMOUNT` DOUBLE,`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," + + "`BATCH_TIME_IN` DATETIME,`BATCH_TIME_OUT` DATETIME,PRIMARY KEY (`ID`, `NAME`, `BATCH_TIME_IN`))"; + + public void assertIfListsAreSameIgnoringOrder(List first, List second) + { + assertTrue(first.size() == second.size() && + first.stream().sorted().collect(Collectors.toList()) + .equals(second.stream().sorted().collect(Collectors.toList()))); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaTest.java new file mode 100644 index 00000000000..c0f7b2a0023 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaTest.java @@ -0,0 +1,350 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.AnsiTestArtifacts; +import org.finos.legend.engine.persistence.components.common.StatisticName; +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.DataSplitRange; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.junit.jupiter.api.Assertions; + +import java.util.ArrayList; +import java.util.List; + +public class NontemporalDeltaTest extends org.finos.legend.engine.persistence.components.ingestmode.nontemporal.NontemporalDeltaTest +{ + + protected String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + protected String incomingRecordCountWithSplits = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE " + + "(stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + protected String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + protected String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + protected String rowsDeletedWithDeleteIndicator = "SELECT COUNT(*) as `rowsDeleted` FROM `mydb`.`main` as sink WHERE EXISTS (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest` FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`) AND (stage.`delete_indicator` IN ('yes','1','true')))"; + + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + @Override + public void verifyNontemporalDeltaNoAuditingNoDataSplit(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING `mydb`.`staging` as stage " + + "ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND sink.`digest` <> stage.`digest` " + + "THEN UPDATE SET " + + "sink.`id` = stage.`id`," + + "sink.`name` = stage.`name`," + + "sink.`amount` = stage.`amount`," + + "sink.`biz_date` = stage.`biz_date`," + + "sink.`digest` = stage.`digest` " + + "WHEN NOT MATCHED THEN " + + "INSERT (`id`, `name`, `amount`, `biz_date`, `digest`) " + + "VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + + // Stats + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyNontemporalDeltaWithAuditingNoDataSplit(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING `mydb`.`staging` as stage " + + "ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND sink.`digest` <> stage.`digest` " + + "THEN UPDATE SET " + + "sink.`id` = stage.`id`," + + "sink.`name` = stage.`name`," + + "sink.`amount` = stage.`amount`," + + "sink.`biz_date` = stage.`biz_date`," + + "sink.`digest` = stage.`digest`," + + "sink.`batch_update_time` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHEN NOT MATCHED THEN INSERT " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_update_time`) " + + "VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestPlusUpdateTimestampCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + + // Stats + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyNonTemporalDeltaNoAuditingWithDataSplit(List operations, List dataSplitRanges) + { + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest` FROM `mydb`.`staging` as stage " + + "WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) " + + "as stage ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND sink.`digest` <> stage.`digest` " + + "THEN UPDATE SET sink.`id` = stage.`id`,sink.`name` = stage.`name`,sink.`amount` = stage.`amount`,sink.`biz_date` = stage.`biz_date`,sink.`digest` = stage.`digest` " + + "WHEN NOT MATCHED " + + "THEN INSERT (`id`, `name`, `amount`, `biz_date`, `digest`) " + + "VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(mergeSql, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(mergeSql, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + + // Stats + Assertions.assertEquals(enrichSqlWithDataSplits(incomingRecordCountWithSplits, dataSplitRanges.get(0)), operations.get(0).postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(enrichSqlWithDataSplits(incomingRecordCountWithSplits, dataSplitRanges.get(1)), operations.get(1).postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsTerminated, operations.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyNonTemporalDeltaWithWithAuditingWithDataSplit(List operations, List dataSplitRanges) + { + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest` FROM `mydb`.`staging` as stage " + + "WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) " + + "as stage ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND sink.`digest` <> stage.`digest` " + + "THEN UPDATE SET sink.`id` = stage.`id`,sink.`name` = stage.`name`,sink.`amount` = stage.`amount`,sink.`biz_date` = stage.`biz_date`,sink.`digest` = stage.`digest`,sink.`batch_update_time` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHEN NOT MATCHED " + + "THEN INSERT (`id`, `name`, `amount`, `biz_date`, `digest`, `batch_update_time`) " + + "VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestPlusUpdateTimestampCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(mergeSql, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(mergeSql, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + + // Stats + Assertions.assertEquals(enrichSqlWithDataSplits(incomingRecordCountWithSplits, dataSplitRanges.get(0)), operations.get(0).postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(enrichSqlWithDataSplits(incomingRecordCountWithSplits, dataSplitRanges.get(1)), operations.get(1).postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsTerminated, operations.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.get(0).postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyNontemporalDeltaNoAuditingNoDataSplitWithDeleteIndicator(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING `mydb`.`staging` as stage " + + "ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND (sink.`digest` <> stage.`digest`) AND (stage.`delete_indicator` NOT IN ('yes','1','true')) " + + "THEN UPDATE SET " + + "sink.`id` = stage.`id`," + + "sink.`name` = stage.`name`," + + "sink.`amount` = stage.`amount`," + + "sink.`biz_date` = stage.`biz_date`," + + "sink.`digest` = stage.`digest` " + + "WHEN NOT MATCHED THEN " + + "INSERT (`id`, `name`, `amount`, `biz_date`, `digest`) " + + "VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + + // Stats + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(null, operations.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + Assertions.assertEquals(rowsDeletedWithDeleteIndicator, operations.preIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyNontemporalDeltaWithUpperCaseOptimizer(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `MYDB`.`MAIN` as sink USING `MYDB`.`STAGING` as stage " + + "ON (sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`) WHEN MATCHED " + + "AND sink.`DIGEST` <> stage.`DIGEST` THEN UPDATE SET sink.`ID` = stage.`ID`," + + "sink.`NAME` = stage.`NAME`,sink.`AMOUNT` = stage.`AMOUNT`," + + "sink.`BIZ_DATE` = stage.`BIZ_DATE`,sink.`DIGEST` = stage.`DIGEST` " + + "WHEN NOT MATCHED THEN INSERT (`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`) " + + "VALUES (stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQueryWithUpperCase, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + } + + @Override + public void verifyNontemporalDeltaWithLessColumnsInStaging(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING `mydb`.`staging` as stage " + + "ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND sink.`digest` <> stage.`digest` " + + "THEN UPDATE SET " + + "sink.`id` = stage.`id`,sink.`name` = stage.`name`,sink.`amount` = stage.`amount`,sink.`digest` = stage.`digest` " + + "WHEN NOT MATCHED THEN INSERT " + + "(`id`, `name`, `amount`, `digest`) " + + "VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`digest`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + } + + @Override + public void verifyNontemporalDeltaWithNoVersionAndStagingFilter(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest` FROM `mydb`.`staging` as stage WHERE (stage.`biz_date` > '2020-01-01') AND (stage.`biz_date` < '2020-01-03')) as stage " + + "ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND sink.`digest` <> stage.`digest` " + + "THEN UPDATE SET " + + "sink.`id` = stage.`id`," + + "sink.`name` = stage.`name`," + + "sink.`amount` = stage.`amount`," + + "sink.`biz_date` = stage.`biz_date`," + + "sink.`digest` = stage.`digest` " + + "WHEN NOT MATCHED THEN " + + "INSERT (`id`, `name`, `amount`, `biz_date`, `digest`) " + + "VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`biz_date` > '2020-01-01') AND (stage.`biz_date` < '2020-01-03')"; + // Stats + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyNontemporalDeltaWithMaxVersioningAndStagingFiltersWithDedup(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version` FROM " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`,ROW_NUMBER() OVER (PARTITION BY stage.`id`,stage.`name` ORDER BY stage.`version` DESC) as `legend_persistence_row_num` FROM `mydb`.`staging` as stage WHERE stage.`snapshot_id` > 18972) as stage WHERE stage.`legend_persistence_row_num` = 1) as stage " + + "ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND stage.`version` > sink.`version` " + + "THEN UPDATE SET sink.`id` = stage.`id`,sink.`name` = stage.`name`,sink.`amount` = stage.`amount`,sink.`biz_date` = stage.`biz_date`,sink.`digest` = stage.`digest`,sink.`version` = stage.`version` " + + "WHEN NOT MATCHED THEN INSERT (`id`, `name`, `amount`, `biz_date`, `digest`, `version`) VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestPlusVersionCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE stage.`snapshot_id` > 18972"; + // Stats + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyNontemporalDeltaWithMaxVersioningNoDedupAndStagingFilters(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version` FROM `mydb`.`staging` as stage WHERE stage.`snapshot_id` > 18972) as stage " + + "ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND stage.`version` > sink.`version` " + + "THEN UPDATE SET sink.`id` = stage.`id`,sink.`name` = stage.`name`,sink.`amount` = stage.`amount`,sink.`biz_date` = stage.`biz_date`,sink.`digest` = stage.`digest`,sink.`version` = stage.`version` " + + "WHEN NOT MATCHED THEN INSERT (`id`, `name`, `amount`, `biz_date`, `digest`, `version`) VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestPlusVersionCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE stage.`snapshot_id` > 18972"; + // Stats + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyNontemporalDeltaWithMaxVersioningNoDedupWithoutStagingFilters(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `mydb`.`main` as sink " + + "USING " + + "`mydb`.`staging` as stage " + + "ON (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`) " + + "WHEN MATCHED AND stage.`version` > sink.`version` " + + "THEN UPDATE SET sink.`id` = stage.`id`,sink.`name` = stage.`name`,sink.`amount` = stage.`amount`,sink.`biz_date` = stage.`biz_date`,sink.`digest` = stage.`digest`,sink.`version` = stage.`version` " + + "WHEN NOT MATCHED THEN INSERT (`id`, `name`, `amount`, `biz_date`, `digest`, `version`) VALUES (stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestPlusVersionCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + + // Stats + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + Assertions.assertEquals(rowsDeleted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + } + + @Override + public void verifyNontemporalDeltaWithWithMaxVersioningDedupEnabledAndUpperCaseWithoutStagingFilters(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String mergeSql = "MERGE INTO `MYDB`.`MAIN` as sink " + + "USING " + + "(SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,stage.`VERSION` FROM " + + "(SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,stage.`VERSION`,ROW_NUMBER() OVER (PARTITION BY stage.`ID`,stage.`NAME` ORDER BY stage.`VERSION` DESC) as `LEGEND_PERSISTENCE_ROW_NUM` FROM `MYDB`.`STAGING` as stage) as stage WHERE stage.`LEGEND_PERSISTENCE_ROW_NUM` = 1) as stage " + + "ON (sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`) " + + "WHEN MATCHED AND stage.`VERSION` >= sink.`VERSION` " + + "THEN UPDATE SET sink.`ID` = stage.`ID`,sink.`NAME` = stage.`NAME`,sink.`AMOUNT` = stage.`AMOUNT`,sink.`BIZ_DATE` = stage.`BIZ_DATE`,sink.`DIGEST` = stage.`DIGEST`,sink.`VERSION` = stage.`VERSION` " + + "WHEN NOT MATCHED THEN INSERT (`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`, `VERSION`) VALUES (stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,stage.`VERSION`)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTablePlusDigestPlusVersionCreateQueryUpperCase, preActionsSqlList.get(0)); + Assertions.assertEquals(mergeSql, milestoningSqlList.get(0)); + } + + @Override + public void verifyNontemporalDeltaPostActionSqlAndCleanStagingData(GeneratorResult operations) + { + List postActionsSql = operations.postActionsSql(); + List expectedSQL = new ArrayList<>(); + expectedSQL.add(BigQueryTestArtifacts.expectedStagingCleanupQuery); + assertIfListsAreSameIgnoringOrder(expectedSQL, postActionsSql); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalSnapshotTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalSnapshotTest.java new file mode 100644 index 00000000000..0991fe0b5e0 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalSnapshotTest.java @@ -0,0 +1,175 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.common.StatisticName; +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.testcases.ingestmode.nontemporal.NontemporalSnapshotTestCases; +import org.junit.jupiter.api.Assertions; + +import java.util.ArrayList; +import java.util.List; + +public class NontemporalSnapshotTest extends NontemporalSnapshotTestCases +{ + String rowsDeleted = "SELECT COUNT(*) as `rowsDeleted` FROM `mydb`.`main` as sink"; + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT 0 as `rowsUpdated`"; + String rowsInserted = "SELECT COUNT(*) as `rowsInserted` FROM `mydb`.`main` as sink"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + + @Override + public void verifyNontemporalSnapshotNoAuditingNoDataSplit(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`) " + + "(SELECT * FROM `mydb`.`staging` as stage)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTableCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.cleanUpMainTableSql, milestoningSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(1)); + + // Stats + verifyStats(operations); + } + + @Override + public void verifyNontemporalSnapshotNoAuditingWithDataSplit(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date` FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`staging` as stage_right WHERE " + + "(stage.`data_split` < stage_right.`data_split`) AND ((stage.`id` = stage_right.`id`) AND (stage.`name` = stage_right.`name`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTableCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.cleanUpMainTableSql, milestoningSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(1)); + + // Stats + verifyStats(operations); + } + + @Override + public void verifyNontemporalSnapshotWithAuditingNoDataSplit(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `batch_update_time`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "FROM `mydb`.`staging` as stage)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTableWithAuditPKCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.cleanUpMainTableSql, milestoningSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(1)); + + // Stats + verifyStats(operations); + } + + @Override + public void verifyNontemporalSnapshotWithAuditingWithDataSplit(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`, `batch_update_time`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "FROM `mydb`.`staging` as stage WHERE NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage_right " + + "WHERE (stage.`data_split` < stage_right.`data_split`) AND ((stage.`id` = stage_right.`id`) AND " + + "(stage.`name` = stage_right.`name`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTableWithAuditPKCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.cleanUpMainTableSql, milestoningSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(1)); + + // Stats + verifyStats(operations); + } + + @Override + public void verifyNontemporalSnapshotWithUpperCaseOptimizer(GeneratorResult queries) + { + List preActionsSqlList = queries.preActionsSql(); + List milestoningSqlList = queries.ingestSql(); + + String insertSql = "INSERT INTO `MYDB`.`MAIN` (`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`) " + + "(SELECT * FROM `MYDB`.`STAGING` as stage)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTableCreateQueryWithUpperCase, preActionsSqlList.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.cleanupMainTableSqlUpperCase, milestoningSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(1)); + } + + @Override + public void verifyNontemporalSnapshotWithLessColumnsInStaging(GeneratorResult operations) + { + List preActionsSqlList = operations.preActionsSql(); + List milestoningSqlList = operations.ingestSql(); + + String insertSql = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`) " + + "(SELECT * FROM `mydb`.`staging` as stage)"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedBaseTableCreateQuery, preActionsSqlList.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.cleanUpMainTableSql, milestoningSqlList.get(0)); + Assertions.assertEquals(insertSql, milestoningSqlList.get(1)); + } + + @Override + public void verifyNontemporalSnapshotWithCleanStagingData(GeneratorResult operations) + { + List postActionsSql = operations.postActionsSql(); + List expectedSQL = new ArrayList<>(); + expectedSQL.add(BigQueryTestArtifacts.expectedStagingCleanupQuery); + assertIfListsAreSameIgnoringOrder(expectedSQL, postActionsSql); + } + + @Override + public void verifyNontemporalSnapshotWithDropStagingData(SqlPlan physicalPlanForPostActions) + { + List sqlsForPostActions = physicalPlanForPostActions.getSqlList(); + List expectedSQL = new ArrayList<>(); + expectedSQL.add(BigQueryTestArtifacts.expectedDropTableQuery); + assertIfListsAreSameIgnoringOrder(expectedSQL, sqlsForPostActions); + } + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + private void verifyStats(GeneratorResult operations) + { + // Pre stats: + Assertions.assertEquals(rowsDeleted, operations.preIngestStatisticsSql().get(StatisticName.ROWS_DELETED)); + + // Post Stats: + Assertions.assertEquals(incomingRecordCount, operations.postIngestStatisticsSql().get(StatisticName.INCOMING_RECORD_COUNT)); + Assertions.assertEquals(rowsUpdated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_UPDATED)); + Assertions.assertEquals(rowsInserted, operations.postIngestStatisticsSql().get(StatisticName.ROWS_INSERTED)); + Assertions.assertEquals(rowsTerminated, operations.postIngestStatisticsSql().get(StatisticName.ROWS_TERMINATED)); + } +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdBasedTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdBasedTest.java new file mode 100644 index 00000000000..c4ed9b295da --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdBasedTest.java @@ -0,0 +1,480 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.DataSplitRange; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.testcases.ingestmode.unitemporal.UnitmemporalDeltaBatchIdBasedTestCases; +import org.junit.jupiter.api.Assertions; + +import java.util.List; + +public class UnitemporalDeltaBatchIdBasedTest extends UnitmemporalDeltaBatchIdBasedTestCases +{ + @Override + public void verifyUnitemporalDeltaNoDeleteIndNoAuditing(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999 " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaNoDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999 " + + "FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND " + + "(NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(1).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + + verifyStats(operations.get(0), enrichSqlWithDataSplits(incomingRecordCount, dataSplitRanges.get(0)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaWithDeleteIndNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = " + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE " + + "(sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) " + + "AND ((sink.`digest` <> stage.`digest`) OR (stage.`delete_indicator` IN ('yes','1','true')))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999 FROM `mydb`.`staging` as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) AND (sink.`digest` = stage.`digest`) " + + "AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) AND " + + "(stage.`delete_indicator` NOT IN ('yes','1','true')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaWithDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = " + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE " + + "(sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) " + + "AND ((sink.`digest` <> stage.`digest`) OR (stage.`delete_indicator` IN ('yes','1','true')))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999 FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND " + + "(NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) AND (sink.`digest` = stage.`digest`) " + + "AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) AND " + + "(stage.`delete_indicator` NOT IN ('yes','1','true')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(1).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + verifyStats(operations.get(0), enrichSqlWithDataSplits(incomingRecordCount, dataSplitRanges.get(0)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaWithUpperCaseOptimizer(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `MYDB`.`MAIN` as sink " + + "SET sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 " + + "FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1 " + + "WHERE (sink.`BATCH_ID_OUT` = 999999999) " + + "AND (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage " + + "WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) " + + "AND (sink.`DIGEST` <> stage.`DIGEST`)))"; + + String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` " + + "(`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`, `BATCH_ID_IN`, `BATCH_ID_OUT`) " + + "(SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`," + + "stage.`DIGEST`,(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')," + + "999999999 FROM `MYDB`.`STAGING` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `MYDB`.`MAIN` as sink " + + "WHERE (sink.`BATCH_ID_OUT` = 999999999) AND (sink.`DIGEST` = stage.`DIGEST`) " + + "AND ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQueryWithUpperCase, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQueryWithUpperCase(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithCleanStagingData(GeneratorResult operations) + { + List postActionsSql = operations.postActionsSql(); + Assertions.assertEquals(BigQueryTestArtifacts.expectedStagingCleanupQuery, postActionsSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaNoDeleteIndNoAuditingWithOptimizationFilters(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 WHERE " + + "(sink.`batch_id_out` = 999999999) AND ((sink.`id` >= '{ID_LOWER_BOUND}') AND (sink.`id` <= '{ID_UPPER_BOUND}')) " + + "AND (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) " + + "AND (sink.`name` = stage.`name`)) AND (sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND " + + "(sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "((sink.`id` >= '{ID_LOWER_BOUND}') AND (sink.`id` <= '{ID_UPPER_BOUND}')))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaNoDeleteIndNoAuditingWithOptimizationFiltersIncludesNullValues(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 WHERE " + + "(sink.`batch_id_out` = 999999999) AND (((sink.`id` >= '{ID_LOWER_BOUND}') AND (sink.`id` <= '{ID_UPPER_BOUND}')) OR (sink.`id` IS NULL)) " + + "AND (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) " + + "AND (sink.`name` = stage.`name`)) AND (sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND " + + "(sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(((sink.`id` >= '{ID_LOWER_BOUND}') AND (sink.`id` <= '{ID_UPPER_BOUND}')) OR (sink.`id` IS NULL)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithNoVersionAndStagingFilter(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE (((sink.`id` = stage.`id`) AND " + + "(sink.`name` = stage.`name`)) AND (sink.`digest` <> stage.`digest`)) AND (stage.`batch_id_in` > 5)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 FROM `mydb`.`staging` as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) " + + "AND (stage.`batch_id_in` > 5))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestWithStagingFiltersQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithMaxVersionDedupEnabledAndStagingFilter(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 " + + "FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) AND (EXISTS " + + "(SELECT * FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version` " + + "FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`,ROW_NUMBER() " + + "OVER (PARTITION BY stage.`id`,stage.`name` ORDER BY stage.`version` DESC) as `legend_persistence_row_num` " + + "FROM `mydb`.`staging` as stage WHERE stage.`batch_id_in` > 5) as stage " + + "WHERE stage.`legend_persistence_row_num` = 1) as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (stage.`version` > sink.`version`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`, " + + "`digest`, `version`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 " + + "FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version` " + + "FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`," + + "ROW_NUMBER() OVER (PARTITION BY stage.`id`,stage.`name` ORDER BY stage.`version` DESC) " + + "as `legend_persistence_row_num` FROM `mydb`.`staging` as stage WHERE stage.`batch_id_in` > 5) as stage " + + "WHERE stage.`legend_persistence_row_num` = 1) as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) AND (stage.`version` <= sink.`version`) " + + "AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdAndVersionBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestWithStagingFiltersQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithMaxVersionNoDedupAndStagingFilter(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE (((sink.`id` = stage.`id`) AND " + + "(sink.`name` = stage.`name`)) AND (stage.`version` > sink.`version`)) AND (stage.`batch_id_in` > 5)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `version`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata " + + "WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 FROM `mydb`.`staging` as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) " + + "AND (stage.`version` <= sink.`version`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) " + + "AND (stage.`batch_id_in` > 5))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdAndVersionBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestWithStagingFiltersQuery(), metadataIngestSql.get(0)); + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE stage.`batch_id_in` > 5"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaWithMaxVersioningNoDedupWithoutStagingFilters(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(stage.`version` > sink.`version`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `version`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999 " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (stage.`version` <= sink.`version`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdAndVersionBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithMaxVersioningDedupEnabledAndUpperCaseWithoutStagingFilters(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + String expectedMilestoneQuery = "UPDATE `MYDB`.`MAIN` as sink SET sink.`BATCH_ID_OUT` = " + + "(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA " + + "as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1 " + + "WHERE (sink.`BATCH_ID_OUT` = 999999999) AND " + + "(EXISTS (SELECT * FROM (SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,stage.`VERSION` " + + "FROM (SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,stage.`VERSION`," + + "ROW_NUMBER() OVER (PARTITION BY stage.`ID`,stage.`NAME` ORDER BY stage.`VERSION` DESC) " + + "as `LEGEND_PERSISTENCE_ROW_NUM` FROM `MYDB`.`STAGING` as stage) as stage WHERE stage.`LEGEND_PERSISTENCE_ROW_NUM` = 1) as stage " + + "WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) AND (stage.`VERSION` >= sink.`VERSION`)))"; + + String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` (`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`, `VERSION`, `BATCH_ID_IN`, `BATCH_ID_OUT`) " + + "(SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,stage.`VERSION`," + + "(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA " + + "WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN'),999999999 FROM " + + "(SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,stage.`VERSION` " + + "FROM (SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,stage.`VERSION`," + + "ROW_NUMBER() OVER (PARTITION BY stage.`ID`,stage.`NAME` ORDER BY stage.`VERSION` DESC) as `LEGEND_PERSISTENCE_ROW_NUM` " + + "FROM `MYDB`.`STAGING` as stage) as stage WHERE stage.`LEGEND_PERSISTENCE_ROW_NUM` = 1) as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `MYDB`.`MAIN` as sink WHERE (sink.`BATCH_ID_OUT` = 999999999) " + + "AND (stage.`VERSION` < sink.`VERSION`) AND ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdAndVersionBasedCreateQueryWithUpperCase, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQueryWithUpperCase(), metadataIngestSql.get(0)); + } + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + protected String getExpectedMetadataTableIngestQuery() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQuery; + } + + protected String getExpectedMetadataTableIngestWithStagingFiltersQuery() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestWithStagingFiltersQuery; + } + + protected String getExpectedMetadataTableIngestQueryWithUpperCase() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithUpperCase; + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdDateTimeBasedTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdDateTimeBasedTest.java new file mode 100644 index 00000000000..46cc5f16f9c --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdDateTimeBasedTest.java @@ -0,0 +1,462 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.DataSplitRange; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.testcases.ingestmode.unitemporal.UnitmemporalDeltaBatchIdDateTimeBasedTestCases; +import org.junit.jupiter.api.Assertions; + +import java.util.ArrayList; +import java.util.List; + +public class UnitemporalDeltaBatchIdDateTimeBasedTest extends UnitmemporalDeltaBatchIdDateTimeBasedTestCases +{ + @Override + public void verifyUnitemporalDeltaNoDeleteIndNoAuditing(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + // Stats + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaNoDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND " + + "(NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(1).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + // Stats + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + + verifyStats(operations.get(0), enrichSqlWithDataSplits(incomingRecordCount, dataSplitRanges.get(0)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaWithDeleteIndMultiValuesNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = " + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE " + + "(sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) " + + "AND ((sink.`digest` <> stage.`digest`) OR (stage.`delete_indicator` IN ('yes','1','true')))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, " + + "`batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `mydb`.`staging` as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) AND (sink.`digest` = stage.`digest`) " + + "AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) AND " + + "(stage.`delete_indicator` NOT IN ('yes','1','true')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + // Stats + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaWithDeleteIndNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = " + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE " + + "(sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) " + + "AND ((sink.`digest` <> stage.`digest`) OR (stage.`delete_indicator` = true))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, " + + "`batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `mydb`.`staging` as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) AND (sink.`digest` = stage.`digest`) " + + "AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) AND " + + "(stage.`delete_indicator` <> true))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE " + + "((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND " + + "((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "((sink.`digest` <> stage.`digest`) OR (stage.`delete_indicator` IN ('yes','1','true')))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND " + + "(NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND " + + "(sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND " + + "(sink.`name` = stage.`name`))))) AND (stage.`delete_indicator` NOT IN ('yes','1','true')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(1).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + } + + @Override + public void verifyUnitemporalDeltaWithCleanStagingData(GeneratorResult operations) + { + Assertions.assertEquals(0, new ArrayList<>(operations.postIngestStatisticsSql().values()).size()); + List postActionsSql = operations.postActionsSql(); + Assertions.assertEquals(BigQueryTestArtifacts.expectedStagingCleanupQuery, postActionsSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithUpperCaseOptimizer(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `MYDB`.`MAIN` as sink SET sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1,sink.`BATCH_TIME_OUT` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') WHERE (sink.`BATCH_ID_OUT` = 999999999) AND (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) AND (sink.`DIGEST` <> stage.`DIGEST`)))"; + String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` (`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`, `BATCH_ID_IN`, `BATCH_ID_OUT`, `BATCH_TIME_IN`, `BATCH_TIME_OUT`) (SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN'),999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `MYDB`.`STAGING` as stage WHERE NOT (EXISTS (SELECT * FROM `MYDB`.`MAIN` as sink WHERE (sink.`BATCH_ID_OUT` = 999999999) AND (sink.`DIGEST` = stage.`DIGEST`) AND ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)))))"; + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQueryWithUpperCase, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQueryWithUpperCase(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithLessColumnsInStaging(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE " + + "((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) AND (sink.`digest` = stage.`digest`) " + + "AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithPlaceholders(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = {BATCH_ID_PATTERN}-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','{BATCH_START_TS_PATTERN}') " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "{BATCH_ID_PATTERN},999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','{BATCH_START_TS_PATTERN}'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithPlaceHolders, metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithOnlySchemaSet(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedCreateMainTableQuery = "CREATE TABLE IF NOT EXISTS `my_schema`.`main`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`batch_time_in` DATETIME," + + "`batch_time_out` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)"; + + String expectedMilestoneQuery = "UPDATE `my_schema`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `my_schema`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `my_schema`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `my_schema`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `my_schema`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(expectedCreateMainTableQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithDbAndSchemaBothSet(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedCreateMainTableQuery = "CREATE TABLE IF NOT EXISTS `mydb`.`my_schema`.`main`" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`batch_time_in` DATETIME," + + "`batch_time_out` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)"; + + String expectedMilestoneQuery = "UPDATE `mydb`.`my_schema`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM `mydb`.`my_schema`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`my_schema`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`my_schema`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`my_schema`.`main` as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(expectedCreateMainTableQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithDbAndSchemaBothNotSet(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedCreateMainTableQuery = "CREATE TABLE IF NOT EXISTS main" + + "(`id` INT64 NOT NULL," + + "`name` STRING NOT NULL," + + "`amount` FLOAT64," + + "`biz_date` DATE," + + "`digest` STRING," + + "`batch_id_in` INT64 NOT NULL," + + "`batch_id_out` INT64," + + "`batch_time_in` DATETIME," + + "`batch_time_out` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`) NOT ENFORCED)"; + + String expectedMilestoneQuery = "UPDATE main as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1," + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) AND " + + "(EXISTS (SELECT * FROM staging as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO main " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + + "999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM staging as stage " + + "WHERE NOT (EXISTS (SELECT * FROM main as sink " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(expectedCreateMainTableQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + protected String getExpectedMetadataTableIngestQuery() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQuery; + } + + protected String getExpectedMetadataTableIngestQueryWithUpperCase() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithUpperCase; + } + +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaDateTimeBasedTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaDateTimeBasedTest.java new file mode 100644 index 00000000000..d3c8e25dc7d --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaDateTimeBasedTest.java @@ -0,0 +1,232 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.DataSplitRange; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.testcases.ingestmode.unitemporal.UnitmemporalDeltaDateTimeBasedTestCases; +import org.junit.jupiter.api.Assertions; + +import java.util.List; + +public class UnitemporalDeltaDateTimeBasedTest extends UnitmemporalDeltaDateTimeBasedTestCases +{ + @Override + public void verifyUnitemporalDeltaNoDeleteIndNoAuditing(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + // Stats + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaNoDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(sink.`digest` <> stage.`digest`)))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND " + + "(NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(1).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + // Stats + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) as `rowsInserted`"; + String rowsTerminated = "SELECT 0 as `rowsTerminated`"; + verifyStats(operations.get(0), enrichSqlWithDataSplits(incomingRecordCount, dataSplitRanges.get(0)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaWithDeleteIndNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE " + + "(sink.`batch_time_out` = '9999-12-31 23:59:59') AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) " + + "AND ((sink.`digest` <> stage.`digest`) OR (stage.`delete_indicator` IN ('yes','1','true')))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, " + + "`batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `mydb`.`staging` as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') AND (sink.`digest` = stage.`digest`) " + + "AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) AND " + + "(stage.`delete_indicator` NOT IN ('yes','1','true')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + // Stats + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))) as `rowsTerminated`"; + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaWithDeleteIndWithDataSplits(List operations, List dataSplitRanges) + { + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE " + + "(sink.`batch_time_out` = '9999-12-31 23:59:59') AND " + + "(EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) " + + "AND ((sink.`digest` <> stage.`digest`) OR (stage.`delete_indicator` IN ('yes','1','true')))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, " + + "`batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `mydb`.`staging` as stage " + + "WHERE ((stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) AND " + + "(NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') AND (sink.`digest` = stage.`digest`) " + + "AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) AND " + + "(stage.`delete_indicator` NOT IN ('yes','1','true')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQuery, operations.get(0).preActionsSql().get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, operations.get(0).preActionsSql().get(1)); + + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(0)), operations.get(0).ingestSql().get(1)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedMilestoneQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(0)); + Assertions.assertEquals(enrichSqlWithDataSplits(expectedUpsertQuery, dataSplitRanges.get(1)), operations.get(1).ingestSql().get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(0).metadataIngestSql().get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), operations.get(1).metadataIngestSql().get(0)); + Assertions.assertEquals(2, operations.size()); + + // Stats + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage WHERE (stage.`data_split` >= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.`data_split` <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))) as `rowsTerminated`"; + verifyStats(operations.get(0), enrichSqlWithDataSplits(incomingRecordCount, dataSplitRanges.get(0)), rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalDeltaWithUpperCaseOptimizer(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `MYDB`.`MAIN` as sink SET sink.`BATCH_TIME_OUT` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') WHERE (sink.`BATCH_TIME_OUT` = '9999-12-31 23:59:59') AND (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) AND (sink.`DIGEST` <> stage.`DIGEST`)))"; + + String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` (`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`, `BATCH_TIME_IN`, `BATCH_TIME_OUT`) (SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `MYDB`.`STAGING` as stage WHERE NOT (EXISTS (SELECT * FROM `MYDB`.`MAIN` as sink WHERE (sink.`BATCH_TIME_OUT` = '9999-12-31 23:59:59') AND (sink.`DIGEST` = stage.`DIGEST`) AND ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQueryWithUpperCase, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQueryWithUpperCase(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalDeltaWithCleanStagingData(GeneratorResult operations) + { + List postActionsSql = operations.postActionsSql(); + Assertions.assertEquals(BigQueryTestArtifacts.expectedStagingCleanupQuery, postActionsSql.get(0)); + } + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + protected String getExpectedMetadataTableIngestQuery() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQuery; + } + + protected String getExpectedMetadataTableIngestQueryWithUpperCase() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithUpperCase; + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java new file mode 100644 index 00000000000..e312c0ae321 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdBasedTest.java @@ -0,0 +1,177 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.testcases.ingestmode.unitemporal.UnitmemporalSnapshotBatchIdBasedTestCases; +import org.junit.jupiter.api.Assertions; + +import java.util.List; + +public class UnitemporalSnapshotBatchIdBasedTest extends UnitmemporalSnapshotBatchIdBasedTestCases +{ + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + + @Override + public void verifyUnitemporalSnapshotWithoutPartitionNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999)))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalSnapshotWithoutPartitionForEmptyBatch(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE sink.`batch_id_out` = 999999999"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithoutPartitionWithUpperCaseOptimizer(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `MYDB`.`MAIN` as sink SET sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1 WHERE (sink.`BATCH_ID_OUT` = 999999999) AND (NOT (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) AND (sink.`DIGEST` = stage.`DIGEST`))))"; + String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` (`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`, `BATCH_ID_IN`, `BATCH_ID_OUT`) (SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN'),999999999 FROM `MYDB`.`STAGING` as stage WHERE NOT (stage.`DIGEST` IN (SELECT sink.`DIGEST` FROM `MYDB`.`MAIN` as sink WHERE sink.`BATCH_ID_OUT` = 999999999)))"; + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQueryWithUpperCase, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQueryWithUpperCase(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithPartitionNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`)))) " + + "AND (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE sink.`biz_date` = stage.`biz_date`))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND (sink.`biz_date` = stage.`biz_date`))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalSnapshotWithPartitionFiltersNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`)))) " + + "AND (sink.`biz_date` IN ('2000-01-01 00:00:00','2000-01-02 00:00:00'))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND " + + "(sink.`biz_date` IN ('2000-01-01 00:00:00','2000-01-02 00:00:00')))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithCleanStagingData(GeneratorResult operations) + { + List postActionsSql = operations.postActionsSql(); + Assertions.assertEquals(BigQueryTestArtifacts.expectedStagingCleanupQuery, postActionsSql.get(0)); + } + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + protected String getExpectedMetadataTableIngestQuery() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQuery; + } + + protected String getExpectedMetadataTableIngestQueryWithUpperCase() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithUpperCase; + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdDateTimeBasedTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdDateTimeBasedTest.java new file mode 100644 index 00000000000..32c87325830 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotBatchIdDateTimeBasedTest.java @@ -0,0 +1,233 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.testcases.ingestmode.unitemporal.UnitmemporalSnapshotBatchIdDateTimeBasedTestCases; +import org.junit.jupiter.api.Assertions; + +import java.util.List; + +public class UnitemporalSnapshotBatchIdDateTimeBasedTest extends UnitmemporalSnapshotBatchIdDateTimeBasedTestCases +{ + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1)-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_id_in` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'))))) as `rowsTerminated`"; + + @Override + public void verifyUnitemporalSnapshotWithoutPartitionNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1,sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999)))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalSnapshotWithoutPartitionForEmptyBatch(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1,sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE sink.`batch_id_out` = 999999999"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithoutPartitionWithUpperCaseOptimizer(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `MYDB`.`MAIN` as sink SET sink.`BATCH_ID_OUT` = (SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN')-1,sink.`BATCH_TIME_OUT` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') WHERE (sink.`BATCH_ID_OUT` = 999999999) AND (NOT (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage WHERE ((sink.`ID` = stage.`ID`) AND (sink.`NAME` = stage.`NAME`)) AND (sink.`DIGEST` = stage.`DIGEST`))))"; + String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` (`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`, `BATCH_ID_IN`, `BATCH_ID_OUT`, `BATCH_TIME_IN`, `BATCH_TIME_OUT`) (SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`,(SELECT COALESCE(MAX(BATCH_METADATA.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as BATCH_METADATA WHERE UPPER(BATCH_METADATA.`TABLE_NAME`) = 'MAIN'),999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `MYDB`.`STAGING` as stage WHERE NOT (stage.`DIGEST` IN (SELECT sink.`DIGEST` FROM `MYDB`.`MAIN` as sink WHERE sink.`BATCH_ID_OUT` = 999999999)))"; + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQueryWithUpperCase, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQueryWithUpperCase(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithPartitionNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1,sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`)))) " + + "AND (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE sink.`biz_date` = stage.`biz_date`))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND (sink.`biz_date` = stage.`biz_date`))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalSnapshotWithPartitionFiltersNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1,sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`)))) " + + "AND (sink.`biz_date` IN ('2000-01-01 00:00:00','2000-01-02 00:00:00'))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND (sink.`biz_date` IN ('2000-01-01 00:00:00','2000-01-02 00:00:00')))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithCleanStagingData(GeneratorResult operations) + { + List postActionsSql = operations.postActionsSql(); + Assertions.assertEquals(BigQueryTestArtifacts.expectedStagingCleanupQuery, postActionsSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithLessColumnsInStaging(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1,sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`digest`," + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999)))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithPlaceholders(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_id_out` = {BATCH_ID_PATTERN}-1,sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','{BATCH_START_TS_PATTERN}') " + + "WHERE (sink.`batch_id_out` = 999999999) " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "{BATCH_ID_PATTERN},999999999,PARSE_DATETIME('%Y-%m-%d %H:%M:%S','{BATCH_START_TS_PATTERN}'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE sink.`batch_id_out` = 999999999)))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithPlaceHolders, metadataIngestSql.get(0)); + } + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + protected String getExpectedMetadataTableIngestQuery() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQuery; + } + + protected String getExpectedMetadataTableIngestQueryWithUpperCase() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithUpperCase; + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotDateTimeBasedTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotDateTimeBasedTest.java new file mode 100644 index 00000000000..79621de87fb --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalSnapshotDateTimeBasedTest.java @@ -0,0 +1,193 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.ingestmode; + +import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.api.GeneratorResult; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.testcases.ingestmode.unitemporal.UnitmemporalSnapshotDateTimeBasedTestCases; +import org.junit.jupiter.api.Assertions; + +import java.util.List; + +public class UnitemporalSnapshotDateTimeBasedTest extends UnitmemporalSnapshotDateTimeBasedTestCases +{ + + String incomingRecordCount = "SELECT COUNT(*) as `incomingRecordCount` FROM `mydb`.`staging` as stage"; + String rowsUpdated = "SELECT COUNT(*) as `rowsUpdated` FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))"; + String rowsDeleted = "SELECT 0 as `rowsDeleted`"; + String rowsInserted = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))) as `rowsInserted`"; + String rowsTerminated = "SELECT (SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))-(SELECT COUNT(*) FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00')) AND (EXISTS (SELECT * FROM `mydb`.`main` as sink2 WHERE ((sink2.`id` = sink.`id`) AND (sink2.`name` = sink.`name`)) AND (sink2.`batch_time_in` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'))))) as `rowsTerminated`"; + + @Override + public void verifyUnitemporalSnapshotWithoutPartitionNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage " + + "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`))))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE sink.`batch_time_out` = '9999-12-31 23:59:59')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalSnapshotWithoutPartitionForEmptyBatch(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE sink.`batch_time_out` = '9999-12-31 23:59:59'"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithoutPartitionWithUpperCaseOptimizer(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `MYDB`.`MAIN` as sink SET " + + "sink.`BATCH_TIME_OUT` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`BATCH_TIME_OUT` = '9999-12-31 23:59:59') AND " + + "(NOT (EXISTS (SELECT * FROM `MYDB`.`STAGING` as stage WHERE ((sink.`ID` = stage.`ID`) " + + "AND (sink.`NAME` = stage.`NAME`)) AND (sink.`DIGEST` = stage.`DIGEST`))))"; + + String expectedUpsertQuery = "INSERT INTO `MYDB`.`MAIN` " + + "(`ID`, `NAME`, `AMOUNT`, `BIZ_DATE`, `DIGEST`, `BATCH_TIME_IN`, `BATCH_TIME_OUT`) " + + "(SELECT stage.`ID`,stage.`NAME`,stage.`AMOUNT`,stage.`BIZ_DATE`,stage.`DIGEST`," + + "PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `MYDB`.`STAGING` as stage " + + "WHERE NOT (stage.`DIGEST` IN (SELECT sink.`DIGEST` FROM `MYDB`.`MAIN` as sink " + + "WHERE sink.`BATCH_TIME_OUT` = '9999-12-31 23:59:59')))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQueryWithUpperCase, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQueryWithUpperCase, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQueryWithUpperCase(), metadataIngestSql.get(0)); + } + + @Override + public void verifyUnitemporalSnapshotWithPartitionNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + + "SET sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') " + + "AND (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`)))) " + + "AND (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE sink.`biz_date` = stage.`biz_date`))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' " + + "FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') AND (sink.`biz_date` = stage.`biz_date`))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalSnapshotWithPartitionFiltersNoDataSplits(GeneratorResult operations) + { + List preActionsSql = operations.preActionsSql(); + List milestoningSql = operations.ingestSql(); + List metadataIngestSql = operations.metadataIngestSql(); + + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET " + + "sink.`batch_time_out` = PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00') " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') AND " + + "(NOT (EXISTS (SELECT * FROM `mydb`.`staging` as stage WHERE ((sink.`id` = stage.`id`) AND " + + "(sink.`name` = stage.`name`)) AND (sink.`digest` = stage.`digest`)))) AND " + + "(sink.`biz_date` IN ('2000-01-01 00:00:00','2000-01-02 00:00:00'))"; + + String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_time_in`, `batch_time_out`) " + + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + + "PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),'9999-12-31 23:59:59' FROM `mydb`.`staging` as stage " + + "WHERE NOT (stage.`digest` IN (SELECT sink.`digest` FROM `mydb`.`main` as sink " + + "WHERE (sink.`batch_time_out` = '9999-12-31 23:59:59') AND " + + "(sink.`biz_date` IN ('2000-01-01 00:00:00','2000-01-02 00:00:00')))))"; + + Assertions.assertEquals(BigQueryTestArtifacts.expectedMainTableTimeBasedCreateQuery, preActionsSql.get(0)); + Assertions.assertEquals(BigQueryTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); + + Assertions.assertEquals(expectedMilestoneQuery, milestoningSql.get(0)); + Assertions.assertEquals(expectedUpsertQuery, milestoningSql.get(1)); + Assertions.assertEquals(getExpectedMetadataTableIngestQuery(), metadataIngestSql.get(0)); + verifyStats(operations, incomingRecordCount, rowsUpdated, rowsDeleted, rowsInserted, rowsTerminated); + } + + @Override + public void verifyUnitemporalSnapshotWithCleanStagingData(GeneratorResult operations) + { + List postActionsSql = operations.postActionsSql(); + Assertions.assertEquals(BigQueryTestArtifacts.expectedStagingCleanupQuery, postActionsSql.get(0)); + } + + @Override + public RelationalSink getRelationalSink() + { + return BigQuerySink.get(); + } + + protected String getExpectedMetadataTableIngestQuery() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQuery; + } + + protected String getExpectedMetadataTableIngestQueryWithUpperCase() + { + return BigQueryTestArtifacts.expectedMetadataTableIngestQueryWithUpperCase; + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/AlterTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/AlterTest.java new file mode 100644 index 00000000000..cdfcdfcf3d9 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/AlterTest.java @@ -0,0 +1,97 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.logicalplan.operations; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.UpperCaseOptimizer; +import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; +import org.finos.legend.engine.persistence.components.transformer.TransformOptions; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithAllColumns; + +public class AlterTest +{ + + @Test + public void testAlterTable() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithAllColumns) + .build(); + Field column = Field.builder().name("column").type(FieldType.of(DataType.STRING, 64, null)).nullable(true).build(); + + Operation add = Alter.of(dataset, Alter.AlterOperation.ADD, column, Optional.empty()); + Operation changeDatatype = Alter.of(dataset, Alter.AlterOperation.CHANGE_DATATYPE, column, Optional.empty()); + Operation nullableColumn = Alter.of(dataset, Alter.AlterOperation.NULLABLE_COLUMN, column, Optional.empty()); + + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(add, changeDatatype, nullableColumn).build(); + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + + String expectedAdd = "ALTER TABLE `my_db`.`my_schema`.`my_table` ADD COLUMN `column` STRING(64)"; + String expectedChangeDataType = "ALTER TABLE `my_db`.`my_schema`.`my_table` ALTER COLUMN `column` SET DATA TYPE STRING(64)"; + String expectedNullableColumn = "ALTER TABLE `my_db`.`my_schema`.`my_table` ALTER COLUMN `column` DROP NOT NULL"; + + Assertions.assertEquals(expectedAdd, list.get(0)); + Assertions.assertEquals(expectedChangeDataType, list.get(1)); + Assertions.assertEquals(expectedNullableColumn, list.get(2)); + } + + @Test + public void testAlterTableWithUpperCase() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithAllColumns) + .build(); + Field column = Field.builder().name("column").type(FieldType.of(DataType.STRING, 64, null)).nullable(true).build(); + + Operation add = Alter.of(dataset, Alter.AlterOperation.ADD, column, Optional.empty()); + Operation changeDatatype = Alter.of(dataset, Alter.AlterOperation.CHANGE_DATATYPE, column, Optional.empty()); + Operation nullableColumn = Alter.of(dataset, Alter.AlterOperation.NULLABLE_COLUMN, column, Optional.empty()); + + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(add, changeDatatype, nullableColumn).build(); + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get(), TransformOptions.builder().addOptimizers(new UpperCaseOptimizer()).build()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + + String expectedAdd = "ALTER TABLE `MY_DB`.`MY_SCHEMA`.`MY_TABLE` ADD COLUMN `COLUMN` STRING(64)"; + String expectedChangeDataType = "ALTER TABLE `MY_DB`.`MY_SCHEMA`.`MY_TABLE` ALTER COLUMN `COLUMN` SET DATA TYPE STRING(64)"; + String expectedNullableColumn = "ALTER TABLE `MY_DB`.`MY_SCHEMA`.`MY_TABLE` ALTER COLUMN `COLUMN` DROP NOT NULL"; + + Assertions.assertEquals(expectedAdd, list.get(0)); + Assertions.assertEquals(expectedChangeDataType, list.get(1)); + Assertions.assertEquals(expectedNullableColumn, list.get(2)); + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/CreateTableTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/CreateTableTest.java new file mode 100644 index 00000000000..6ce708453b3 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/CreateTableTest.java @@ -0,0 +1,256 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.logicalplan.operations; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.UpperCaseOptimizer; +import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; +import org.finos.legend.engine.persistence.components.transformer.TransformOptions; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithAllColumns; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithClusteringAndPartitionKey; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithClusteringKey; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithPartitionKey; + +public class CreateTableTest +{ + + @Test + public void testCreateTable() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithAllColumns) + .build(); + Operation create = Create.of(true, dataset); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + String expected = "CREATE TABLE IF NOT EXISTS `my_db`.`my_schema`.`my_table`(" + + "`col_int` INT64 NOT NULL," + + "`col_integer` INT64 NOT NULL," + + "`col_bigint` INT64," + + "`col_tinyint` INT64," + + "`col_smallint` INT64," + + "`col_int64` INT64," + + "`col_number` NUMERIC," + + "`col_numeric` NUMERIC," + + "`col_numeric_with_precision` NUMERIC(29)," + + "`col_numeric_with_scale` NUMERIC(33,4)," + + "`col_decimal` NUMERIC," + + "`col_real` FLOAT64," + + "`col_float` FLOAT64," + + "`col_double` FLOAT64," + + "`col_float64` FLOAT64," + + "`col_char` STRING," + + "`col_character` STRING," + + "`col_varchar` STRING," + + "`col_longvarchar` STRING," + + "`col_longtext` STRING," + + "`col_text` STRING," + + "`col_string` STRING," + + "`col_string_with_length` STRING(16)," + + "`col_binary` BYTES," + + "`col_varbinary` BYTES," + + "`col_longvarbinary` BYTES," + + "`col_bytes` BYTES," + + "`col_bytes_with_length` BYTES(10)," + + "`col_date` DATE NOT NULL," + + "`col_time` TIME," + + "`col_datetime` DATETIME," + + "`col_timestamp` TIMESTAMP," + + "`col_boolean` BOOL," + + "`col_bool` BOOL," + + "`col_json` JSON,PRIMARY KEY (`col_int`, `col_date`) NOT ENFORCED)"; + + Assertions.assertEquals(expected, list.get(0)); + } + + @Test + public void testCreateTableWithUpperCase() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithAllColumns) + .build(); + Operation create = Create.of(true, dataset); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get(), TransformOptions.builder().addOptimizers(new UpperCaseOptimizer()).build()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + String expected = "CREATE TABLE IF NOT EXISTS `MY_DB`.`MY_SCHEMA`.`MY_TABLE`(" + + "`COL_INT` INT64 NOT NULL," + + "`COL_INTEGER` INT64 NOT NULL," + + "`COL_BIGINT` INT64," + + "`COL_TINYINT` INT64," + + "`COL_SMALLINT` INT64," + + "`COL_INT64` INT64," + + "`COL_NUMBER` NUMERIC," + + "`COL_NUMERIC` NUMERIC," + + "`COL_NUMERIC_WITH_PRECISION` NUMERIC(29)," + + "`COL_NUMERIC_WITH_SCALE` NUMERIC(33,4)," + + "`COL_DECIMAL` NUMERIC," + + "`COL_REAL` FLOAT64," + + "`COL_FLOAT` FLOAT64," + + "`COL_DOUBLE` FLOAT64," + + "`COL_FLOAT64` FLOAT64," + + "`COL_CHAR` STRING," + + "`COL_CHARACTER` STRING," + + "`COL_VARCHAR` STRING," + + "`COL_LONGVARCHAR` STRING," + + "`COL_LONGTEXT` STRING," + + "`COL_TEXT` STRING," + + "`COL_STRING` STRING," + + "`COL_STRING_WITH_LENGTH` STRING(16)," + + "`COL_BINARY` BYTES," + + "`COL_VARBINARY` BYTES," + + "`COL_LONGVARBINARY` BYTES," + + "`COL_BYTES` BYTES," + + "`COL_BYTES_WITH_LENGTH` BYTES(10)," + + "`COL_DATE` DATE NOT NULL," + + "`COL_TIME` TIME," + + "`COL_DATETIME` DATETIME," + + "`COL_TIMESTAMP` TIMESTAMP," + + "`COL_BOOLEAN` BOOL," + + "`COL_BOOL` BOOL," + + "`COL_JSON` JSON," + + "PRIMARY KEY (`COL_INT`, `COL_DATE`) NOT ENFORCED)"; + + Assertions.assertEquals(expected, list.get(0)); + } + + @Test + public void testCreateTableWithClusteringKey() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithClusteringKey) + .build(); + Operation create = Create.of(true, dataset); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + String expected = "CREATE TABLE IF NOT EXISTS `my_db`.`my_schema`.`my_table`(" + + "`col_int` INT64 NOT NULL PRIMARY KEY NOT ENFORCED," + + "`col_integer` INT64 NOT NULL," + + "`col_string` STRING," + + "`col_timestamp` TIMESTAMP," + + "`col_double` FLOAT64) " + + "CLUSTER BY `col_timestamp`,`col_int`"; + + Assertions.assertEquals(expected, list.get(0)); + } + + @Test + public void testCreateTableWithClusteringKeyWithUpperCase() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithClusteringKey) + .build(); + Operation create = Create.of(true, dataset); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get(), TransformOptions.builder().addOptimizers(new UpperCaseOptimizer()).build()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + String expected = "CREATE TABLE IF NOT EXISTS `MY_DB`.`MY_SCHEMA`.`MY_TABLE`(" + + "`COL_INT` INT64 NOT NULL PRIMARY KEY NOT ENFORCED," + + "`COL_INTEGER` INT64 NOT NULL," + + "`COL_STRING` STRING," + + "`COL_TIMESTAMP` TIMESTAMP," + + "`COL_DOUBLE` FLOAT64) " + + "CLUSTER BY `COL_TIMESTAMP`,`COL_INT`"; + + Assertions.assertEquals(expected, list.get(0)); + } + + @Test + public void testCreateTableWithPartitionKeyAndClusteringKey() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithClusteringAndPartitionKey) + .build(); + Operation create = Create.of(true, dataset); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + String expected = "CREATE TABLE IF NOT EXISTS `my_db`.`my_schema`.`my_table`(" + + "`col_int` INT64 NOT NULL," + + "`col_date` DATE NOT NULL," + + "`col_integer` INT64 NOT NULL," + + "`col_string` STRING," + + "`col_timestamp` TIMESTAMP," + + "`col_double` FLOAT64," + + "PRIMARY KEY (`col_int`, `col_date`) NOT ENFORCED) " + + "PARTITION BY `col_date` " + + "CLUSTER BY `col_timestamp`,`col_int`"; + Assertions.assertEquals(expected, list.get(0)); + } + + @Test + public void testCreateTableWithPartitionKey() + { + DatasetDefinition dataset = DatasetDefinition.builder() + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithPartitionKey) + .build(); + Operation create = Create.of(true, dataset); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get()); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + String expected = "CREATE TABLE IF NOT EXISTS `my_db`.`my_schema`.`my_table`(" + + "`col_int` INT64 NOT NULL," + + "`col_date` DATE NOT NULL," + + "`col_integer` INT64 NOT NULL," + + "`col_string` STRING," + + "`col_timestamp` TIMESTAMP," + + "`col_double` FLOAT64," + + "PRIMARY KEY (`col_int`, `col_date`) NOT ENFORCED) " + + "PARTITION BY _PARTITIONDATE"; + Assertions.assertEquals(expected, list.get(0)); + } + + +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/DataTypeMappingTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/DataTypeMappingTest.java new file mode 100644 index 00000000000..2caf5f5b822 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/DataTypeMappingTest.java @@ -0,0 +1,109 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.bigquery.sql; + +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlDomException; +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBigint; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBinary; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBool; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBoolean; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBytes; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBytesWithLength; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colChar; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colCharacter; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDate; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDatetime; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDecimal; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDouble; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colFloat; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colFloat64; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colInt; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colInt64; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colInteger; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colJson; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colLongVarBinary; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colLongVarchar; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colLongtext; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colNumber; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colNumeric; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colNumericWithPrecision; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colNumericWithScale; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colReal; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colSmallint; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colString; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colStringWithLength; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colText; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colTime; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colTimestamp; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colTinyint; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colVarBinary; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colVarchar; + +public class DataTypeMappingTest +{ + + @Test + void testDataType() throws SqlDomException + { + Assertions.assertEquals("INT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colInt.type()))); + Assertions.assertEquals("INT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colInteger.type()))); + Assertions.assertEquals("INT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colBigint.type()))); + Assertions.assertEquals("INT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colTinyint.type()))); + Assertions.assertEquals("INT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colSmallint.type()))); + Assertions.assertEquals("INT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colInt64.type()))); + Assertions.assertEquals("NUMERIC", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colNumber.type()))); + Assertions.assertEquals("NUMERIC", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colNumeric.type()))); + Assertions.assertEquals("NUMERIC", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colDecimal.type()))); + Assertions.assertEquals("NUMERIC(29)", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colNumericWithPrecision.type()))); + Assertions.assertEquals("NUMERIC(33,4)", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colNumericWithScale.type()))); + Assertions.assertEquals("FLOAT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colReal.type()))); + Assertions.assertEquals("FLOAT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colFloat.type()))); + Assertions.assertEquals("FLOAT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colDouble.type()))); + Assertions.assertEquals("FLOAT64", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colFloat64.type()))); + Assertions.assertEquals("STRING", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colChar.type()))); + Assertions.assertEquals("STRING", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colCharacter.type()))); + Assertions.assertEquals("STRING", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colVarchar.type()))); + Assertions.assertEquals("STRING", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colLongVarchar.type()))); + Assertions.assertEquals("STRING", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colLongtext.type()))); + Assertions.assertEquals("STRING", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colText.type()))); + Assertions.assertEquals("STRING", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colString.type()))); + Assertions.assertEquals("STRING(16)", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colStringWithLength.type()))); + Assertions.assertEquals("BYTES", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colBinary.type()))); + Assertions.assertEquals("BYTES", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colVarBinary.type()))); + Assertions.assertEquals("BYTES", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colLongVarBinary.type()))); + Assertions.assertEquals("BYTES", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colBytes.type()))); + Assertions.assertEquals("BYTES(10)", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colBytesWithLength.type()))); + Assertions.assertEquals("DATE", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colDate.type()))); + Assertions.assertEquals("TIME", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colTime.type()))); + Assertions.assertEquals("DATETIME", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colDatetime.type()))); + Assertions.assertEquals("TIMESTAMP", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colTimestamp.type()))); + Assertions.assertEquals("BOOL", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colBoolean.type()))); + Assertions.assertEquals("BOOL", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colBool.type()))); + Assertions.assertEquals("JSON", getGeneratedSql(new BigQueryDataTypeMapping().getDataType(colJson.type()))); + } + + private String getGeneratedSql(SqlGen sqlGen) throws SqlDomException + { + StringBuilder builder = new StringBuilder(); + sqlGen.genSql(builder); + return builder.toString(); + } + + +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/transformer/PlaceholderTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/transformer/PlaceholderTest.java new file mode 100644 index 00000000000..15f815de383 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/transformer/PlaceholderTest.java @@ -0,0 +1,77 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.transformer; + +import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; +import org.finos.legend.engine.persistence.components.logicalplan.values.BatchEndTimestamp; +import org.finos.legend.engine.persistence.components.logicalplan.values.BatchStartTimestamp; +import org.finos.legend.engine.persistence.components.logicalplan.values.StringValue; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; +import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.UpperCaseOptimizer; +import org.finos.legend.engine.persistence.components.relational.bigquery.BigQuerySink; +import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; +import org.finos.legend.engine.persistence.components.util.MetadataDataset; +import org.finos.legend.engine.persistence.components.util.MetadataUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +public class PlaceholderTest +{ + + @Test + void testTimestampPlaceholder() + { + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get(), TransformOptions.builder().batchStartTimestampPattern("{BATCH_START_TIMESTAMP_PLACEHOLDER}").batchEndTimestampPattern("{BATCH_END_TIMESTAMP_PLACEHOLDER}").build()); + MetadataUtils store = new MetadataUtils(MetadataDataset.builder().build()); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps( + store.insertMetaData(StringValue.of("main"), BatchStartTimestamp.INSTANCE, BatchEndTimestamp.INSTANCE)).build(); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + + String expectedSql = "INSERT INTO batch_metadata (`table_name`, `table_batch_id`, `batch_start_ts_utc`, `batch_end_ts_utc`, `batch_status`) (SELECT 'main',(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),PARSE_DATETIME('%Y-%m-%d %H:%M:%S','{BATCH_START_TIMESTAMP_PLACEHOLDER}'),'{BATCH_END_TIMESTAMP_PLACEHOLDER}','DONE')"; + Assertions.assertEquals(expectedSql, list.get(0)); + } + + @Test + void testTimestampPlaceholderWithUpperCase() + { + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get(), TransformOptions.builder().batchStartTimestampPattern("{BATCH_START_TIMESTAMP_PLACEHOLDER}").batchEndTimestampPattern("{BATCH_END_TIMESTAMP_PLACEHOLDER}").addOptimizers(new UpperCaseOptimizer()).build()); + MetadataUtils store = new MetadataUtils(MetadataDataset.builder().build()); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps( + store.insertMetaData(StringValue.of("main"), BatchStartTimestamp.INSTANCE, BatchEndTimestamp.INSTANCE)).build(); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + + String expectedSql = "INSERT INTO BATCH_METADATA (`TABLE_NAME`, `TABLE_BATCH_ID`, `BATCH_START_TS_UTC`, `BATCH_END_TS_UTC`, `BATCH_STATUS`) (SELECT 'main',(SELECT COALESCE(MAX(batch_metadata.`TABLE_BATCH_ID`),0)+1 FROM BATCH_METADATA as batch_metadata WHERE UPPER(batch_metadata.`TABLE_NAME`) = 'MAIN'),PARSE_DATETIME('%Y-%m-%d %H:%M:%S','{BATCH_START_TIMESTAMP_PLACEHOLDER}'),'{BATCH_END_TIMESTAMP_PLACEHOLDER}','DONE')"; + Assertions.assertEquals(expectedSql, list.get(0)); + } + + @Test + void testBatchIdAndTimestampPlaceholder() + { + RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get(), TransformOptions.builder().batchIdPattern("{BATCH_ID_PATTERN}").batchStartTimestampPattern("{BATCH_START_TIMESTAMP_PLACEHOLDER}").batchEndTimestampPattern("{BATCH_END_TIMESTAMP_PLACEHOLDER}").build()); + MetadataUtils store = new MetadataUtils(MetadataDataset.builder().build()); + LogicalPlan logicalPlan = LogicalPlan.builder().addOps( + store.insertMetaData(StringValue.of("main"), BatchStartTimestamp.INSTANCE, BatchEndTimestamp.INSTANCE)).build(); + SqlPlan physicalPlan = transformer.generatePhysicalPlan(logicalPlan); + List list = physicalPlan.getSqlList(); + + String expectedSql = "INSERT INTO batch_metadata (`table_name`, `table_batch_id`, `batch_start_ts_utc`, `batch_end_ts_utc`, `batch_status`) (SELECT 'main',{BATCH_ID_PATTERN},PARSE_DATETIME('%Y-%m-%d %H:%M:%S','{BATCH_START_TIMESTAMP_PLACEHOLDER}'),'{BATCH_END_TIMESTAMP_PLACEHOLDER}','DONE')"; + Assertions.assertEquals(expectedSql, list.get(0)); + } + +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/append/data_pass1.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/append/data_pass1.csv new file mode 100644 index 00000000000..39997a77565 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/append/data_pass1.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00,2000-01-01T00:00:00 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00,2000-01-01T00:00:00 +3,ANDY,3000,2022-12-03,DIGEST3,2023-01-01 00:00:00,2000-01-01T00:00:00 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/append/data_pass2.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/append/data_pass2.csv new file mode 100644 index 00000000000..d9e06c90192 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/append/data_pass2.csv @@ -0,0 +1,5 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00,2000-01-01T00:00:00 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00,2000-01-01T00:00:00 +3,ANDY,3000,2022-12-03,DIGEST3,2023-01-01 00:00:00,2000-01-01T00:00:00 +3,ANDY,3100,2022-12-03,DIGEST3_UPDATED,2023-01-02 00:00:00,2000-01-02T00:00:00 +4,MATT,4000,2022-12-06,DIGEST4,2023-01-02 00:00:00,2000-01-02T00:00:00 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass1.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass1.csv new file mode 100644 index 00000000000..10efde433ba --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass1.csv @@ -0,0 +1 @@ +39915188,1250000,DIGEST1,2022-04-30T00:00:00,9999-12-31T23:59:59,1,999999999 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass2.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass2.csv new file mode 100644 index 00000000000..78cfa1a76b0 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass2.csv @@ -0,0 +1,3 @@ +39915188,1250000,DIGEST1,2022-04-30T00:00:00,9999-12-31T23:59:59,1,1 +39915188,124000,DIGEST2,2022-05-31T00:00:00,9999-12-31T23:59:59,2,999999999 +39915188,1250000,DIGEST1,2022-04-30T00:00:00,2022-05-31T00:00:00,2,999999999 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass3.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass3.csv new file mode 100644 index 00000000000..4c7eb92e481 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass3.csv @@ -0,0 +1,5 @@ +39915188,1250000,DIGEST1,2022-04-30T00:00:00,9999-12-31T23:59:59,1,1 +39915188,124000,DIGEST2,2022-05-31T00:00:00,9999-12-31T23:59:59,2,2 +39915188,1250000,DIGEST1,2022-04-30T00:00:00,2022-05-31T00:00:00,2,999999999 +39915188,120000,DIGEST3,2022-09-30T00:00:00,9999-12-31T23:59:59,3,999999999 +39915188,124000,DIGEST2,2022-05-31T00:00:00,2022-09-30T00:00:00,3,999999999 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass4.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass4.csv new file mode 100644 index 00000000000..e956a2c03ea --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass4.csv @@ -0,0 +1,7 @@ +39915188,1250000,DIGEST1,2022-04-30T00:00:00,9999-12-31T23:59:59,1,1 +39915188,124000,DIGEST2,2022-05-31T00:00:00,9999-12-31T23:59:59,2,2 +39915188,1250000,DIGEST1,2022-04-30T00:00:00,2022-05-31T00:00:00,2,999999999 +39915188,124000,DIGEST2,2022-05-31T00:00:00,2022-09-30T00:00:00,3,3 +39915188,120000,DIGEST3,2022-09-30T00:00:00,9999-12-31T23:59:59,3,999999999 +39915188,122000,DIGEST4,2022-06-30T00:00:00,2022-09-30T00:00:00,4,999999999 +39915188,124000,DIGEST2,2022-05-31T00:00:00,2022-06-30T00:00:00,4,999999999 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass5.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass5.csv new file mode 100644 index 00000000000..3f57eca87eb --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass5.csv @@ -0,0 +1,8 @@ +39915188,1250000,DIGEST1,2022-04-30T00:00:00,9999-12-31T23:59:59,1,1 +39915188,124000,DIGEST2,2022-05-31T00:00:00,9999-12-31T23:59:59,2,2 +39915188,1250000,DIGEST1,2022-04-30T00:00:00,2022-05-31T00:00:00,2,999999999 +39915188,124000,DIGEST2,2022-05-31T00:00:00,2022-09-30T00:00:00,3,3 +39915188,120000,DIGEST3,2022-09-30T00:00:00,9999-12-31T23:59:59,3,999999999 +39915188,124000,DIGEST2,2022-05-31T00:00:00,2022-06-30T00:00:00,4,4 +39915188,122000,DIGEST4,2022-06-30T00:00:00,2022-09-30T00:00:00,4,999999999 +39915188,110000,DIGEST2_UPDATED,2022-05-31T00:00:00,2022-06-30T00:00:00,5,999999999 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass6.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass6.csv new file mode 100644 index 00000000000..20a22ddad28 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/bitemp_delta/data_pass6.csv @@ -0,0 +1,9 @@ +39915188,1250000,DIGEST1,2022-04-30T00:00:00,9999-12-31T23:59:59,1,1 +39915188,124000,DIGEST2,2022-05-31T00:00:00,9999-12-31T23:59:59,2,2 +39915188,1250000,DIGEST1,2022-04-30T00:00:00,2022-05-31T00:00:00,2,999999999 +39915188,124000,DIGEST2,2022-05-31T00:00:00,2022-09-30T00:00:00,3,3 +39915188,120000,DIGEST3,2022-09-30T00:00:00,9999-12-31T23:59:59,3,999999999 +39915188,124000,DIGEST2,2022-05-31T00:00:00,2022-06-30T00:00:00,4,4 +39915188,122000,DIGEST4,2022-06-30T00:00:00,2022-09-30T00:00:00,4,999999999 +39915188,110000,DIGEST2_UPDATED,2022-05-31T00:00:00,2022-06-30T00:00:00,5,5 +39915188,110000,DIGEST2_UPDATED,2022-05-31T00:00:00,2022-06-30T00:00:00,6,999999999 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_delta/data_pass1.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_delta/data_pass1.csv new file mode 100644 index 00000000000..39997a77565 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_delta/data_pass1.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00,2000-01-01T00:00:00 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00,2000-01-01T00:00:00 +3,ANDY,3000,2022-12-03,DIGEST3,2023-01-01 00:00:00,2000-01-01T00:00:00 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_delta/data_pass2.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_delta/data_pass2.csv new file mode 100644 index 00000000000..8c5a5fd0a56 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_delta/data_pass2.csv @@ -0,0 +1,4 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00,2000-01-01T00:00:00 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00,2000-01-01T00:00:00 +3,ANDY,3100,2022-12-03,DIGEST3_UPDATED,2023-01-02 00:00:00,2000-01-02T00:00:00 +4,MATT,4000,2022-12-06,DIGEST4,2023-01-02 00:00:00,2000-01-02T00:00:00 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_snapshot/data_pass1.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_snapshot/data_pass1.csv new file mode 100644 index 00000000000..36d40c9cc36 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_snapshot/data_pass1.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00 +3,ANDY,3000,2022-12-03,DIGEST3,2023-01-01 00:00:00 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_snapshot/data_pass2.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_snapshot/data_pass2.csv new file mode 100644 index 00000000000..ab45692cfe2 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/nontemporal_snapshot/data_pass2.csv @@ -0,0 +1,3 @@ +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-02 00:00:00 +3,ANDY,3100,2022-12-03,DIGEST3_UPDATED,2023-01-02 00:00:00 +4,MATT,4000,2022-12-06,DIGEST4,2023-01-02 00:00:00 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_delta/data_pass1.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_delta/data_pass1.csv new file mode 100644 index 00000000000..8d9e82e6fcd --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_delta/data_pass1.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00,1,999999999 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00,1,999999999 +3,ANDY,3000,2022-12-03,DIGEST3,2023-01-01 00:00:00,1,999999999 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_delta/data_pass2.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_delta/data_pass2.csv new file mode 100644 index 00000000000..91da5f186ff --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_delta/data_pass2.csv @@ -0,0 +1,5 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00,1,999999999 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00,1,999999999 +3,ANDY,3000,2022-12-03,DIGEST3,2023-01-01 00:00:00,1,1 +3,ANDY,3100,2022-12-03,DIGEST3_UPDATED,2023-01-02 00:00:00,2,999999999 +4,MATT,4000,2022-12-06,DIGEST4,2023-01-02 00:00:00,2,999999999 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_snapshot/data_pass1.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_snapshot/data_pass1.csv new file mode 100644 index 00000000000..8d9e82e6fcd --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_snapshot/data_pass1.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00,1,999999999 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00,1,999999999 +3,ANDY,3000,2022-12-03,DIGEST3,2023-01-01 00:00:00,1,999999999 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_snapshot/data_pass2.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_snapshot/data_pass2.csv new file mode 100644 index 00000000000..7b6a2c78f26 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/expected/unitemp_snapshot/data_pass2.csv @@ -0,0 +1,5 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00,1,1 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00,1,999999999 +3,ANDY,3000,2022-12-03,DIGEST3,2023-01-01 00:00:00,1,1 +3,ANDY,3100,2022-12-03,DIGEST3_UPDATED,2023-01-02 00:00:00,2,999999999 +4,MATT,4000,2022-12-06,DIGEST4,2023-01-02 00:00:00,2,999999999 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass1.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass1.csv new file mode 100644 index 00000000000..7d4c82a622c --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass1.csv @@ -0,0 +1 @@ +39915188,2022-04-30 00:00:00.0,1250000,DIGEST1 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass2.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass2.csv new file mode 100644 index 00000000000..3ba9f149f8d --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass2.csv @@ -0,0 +1 @@ +39915188,2022-05-31 00:00:00.0,124000,DIGEST2 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass3.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass3.csv new file mode 100644 index 00000000000..66033e2f5e0 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass3.csv @@ -0,0 +1 @@ +39915188,2022-09-30 00:00:00.0,120000,DIGEST3 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass4.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass4.csv new file mode 100644 index 00000000000..510d12ad670 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass4.csv @@ -0,0 +1 @@ +39915188,2022-06-30 00:00:00.0,122000,DIGEST4 diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass5.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass5.csv new file mode 100644 index 00000000000..d275b68c3db --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass5.csv @@ -0,0 +1 @@ +39915188,2022-05-31 00:00:00.0,110000,DIGEST2_UPDATED diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass6.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/bitemp_delta/data_pass6.csv new file mode 100644 index 00000000000..e69de29bb2d diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/data_pass1.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/data_pass1.csv new file mode 100644 index 00000000000..36d40c9cc36 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/data_pass1.csv @@ -0,0 +1,3 @@ +1,HARRY,1000,2022-12-01,DIGEST1,2023-01-01 00:00:00 +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-01 00:00:00 +3,ANDY,3000,2022-12-03,DIGEST3,2023-01-01 00:00:00 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/data_pass2.csv b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/data_pass2.csv new file mode 100644 index 00000000000..ab45692cfe2 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/resources/input/data_pass2.csv @@ -0,0 +1,3 @@ +2,ROBERT,2000,2022-12-02,DIGEST2,2023-01-02 00:00:00 +3,ANDY,3100,2022-12-03,DIGEST3_UPDATED,2023-01-02 00:00:00 +4,MATT,4000,2022-12-06,DIGEST4,2023-01-02 00:00:00 \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/RelationalSink.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/RelationalSink.java index 8f4eb40e6df..f5b1f677054 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/RelationalSink.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/RelationalSink.java @@ -18,8 +18,11 @@ import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanNode; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import org.finos.legend.engine.persistence.components.optimizer.Optimizer; -import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcHelper; +import org.finos.legend.engine.persistence.components.relational.api.RelationalConnection; +import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutionHelper; import org.finos.legend.engine.persistence.components.relational.sql.TabularData; import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; import org.finos.legend.engine.persistence.components.sink.Sink; @@ -114,18 +117,72 @@ public ConstructDatasetFromDatabase constructDatasetFromDatabaseFn() public abstract Optional optimizerForCaseConversion(CaseConversion caseConversion); + public abstract Executor getRelationalExecutor(RelationalConnection connection); + + //evolve to = field to replace main column (datatype) + //evolve from = reference field to compare sizing/nullability requirements + @Override + public Field evolveFieldLength(Field evolveFrom, Field evolveTo) + { + Optional length = evolveTo.type().length(); + Optional scale = evolveTo.type().scale(); + + //If the oldField and newField have a length associated, pick the greater length + if (evolveFrom.type().length().isPresent() && evolveTo.type().length().isPresent()) + { + length = evolveTo.type().length().get() >= evolveFrom.type().length().get() + ? evolveTo.type().length() + : evolveFrom.type().length(); + } + //Allow length evolution from unspecified length only when data types are same. This is to avoid evolution like SMALLINT(6) -> INT(6) or INT -> DOUBLE(6) and allow for DATETIME -> DATETIME(6) + else if (evolveFrom.type().dataType().equals(evolveTo.type().dataType()) + && evolveFrom.type().length().isPresent() && !evolveTo.type().length().isPresent()) + { + length = evolveFrom.type().length(); + } + + //If the oldField and newField have a scale associated, pick the greater scale + if (evolveFrom.type().scale().isPresent() && evolveTo.type().scale().isPresent()) + { + scale = evolveTo.type().scale().get() >= evolveFrom.type().scale().get() + ? evolveTo.type().scale() + : evolveFrom.type().scale(); + } + //Allow scale evolution from unspecified scale only when data types are same. This is to avoid evolution like SMALLINT(6) -> INT(6) or INT -> DOUBLE(6) and allow for DATETIME -> DATETIME(6) + else if (evolveFrom.type().dataType().equals(evolveTo.type().dataType()) + && evolveFrom.type().scale().isPresent() && !evolveTo.type().scale().isPresent()) + { + scale = evolveFrom.type().scale(); + } + return createNewField(evolveTo, evolveFrom, length, scale); + } + + + @Override + public Field createNewField(Field evolveTo, Field evolveFrom, Optional length, Optional scale) + { + FieldType modifiedFieldType = FieldType.of(evolveTo.type().dataType(), length, scale); + boolean nullability = evolveTo.nullable() || evolveFrom.nullable(); + + //todo : how to handle default value, identity, uniqueness ? + return Field.builder().name(evolveTo.name()).primaryKey(evolveTo.primaryKey()) + .fieldAlias(evolveTo.fieldAlias()).nullable(nullability) + .identity(evolveTo.identity()).unique(evolveTo.unique()) + .defaultValue(evolveTo.defaultValue()).type(modifiedFieldType).build(); + } + public interface DatasetExists { - boolean apply(Executor executor, JdbcHelper sink, Dataset dataset); + boolean apply(Executor executor, RelationalExecutionHelper sink, Dataset dataset); } public interface ValidateMainDatasetSchema { - void execute(Executor executor, JdbcHelper sink, Dataset dataset); + void execute(Executor executor, RelationalExecutionHelper sink, Dataset dataset); } public interface ConstructDatasetFromDatabase { - Dataset execute(Executor executor, JdbcHelper sink, String tableName, String schemaName, String databaseName); + Dataset execute(Executor executor, RelationalExecutionHelper sink, String tableName, String schemaName, String databaseName); } } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/RelationalConnection.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/RelationalConnection.java new file mode 100644 index 00000000000..d51c568b9cf --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/RelationalConnection.java @@ -0,0 +1,19 @@ +// Copyright 2022 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.api; + +public interface RelationalConnection +{ +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/RelationalIngestorAbstract.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/RelationalIngestorAbstract.java index 2f055af5572..24b36ef9b4e 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/RelationalIngestorAbstract.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/api/RelationalIngestorAbstract.java @@ -158,12 +158,12 @@ protected TransformOptions transformOptions() // ---------- API ---------- - public IngestorResult ingest(Connection connection, Datasets datasets) + public IngestorResult ingest(RelationalConnection connection, Datasets datasets) { return ingest(connection, datasets, null).stream().findFirst().orElseThrow(IllegalStateException::new); } - public List ingestWithDataSplits(Connection connection, Datasets datasets, List dataSplitRanges) + public List ingestWithDataSplits(RelationalConnection connection, Datasets datasets, List dataSplitRanges) { // Provide the default dataSplit ranges if missing if (dataSplitRanges == null || dataSplitRanges.isEmpty()) @@ -175,13 +175,13 @@ public List ingestWithDataSplits(Connection connection, Datasets // ---------- UTILITY METHODS ---------- - private List ingest(Connection connection, Datasets datasets, List dataSplitRanges) + private List ingest(RelationalConnection connection, Datasets datasets, List dataSplitRanges) { IngestMode enrichedIngestMode = ApiUtils.applyCase(ingestMode(), caseConversion()); Datasets enrichedDatasets = ApiUtils.applyCase(datasets, caseConversion()); Transformer transformer = new RelationalTransformer(relationalSink(), transformOptions()); - Executor executor = new RelationalExecutor(relationalSink(), JdbcHelper.of(connection)); + Executor executor = relationalSink().getRelationalExecutor(connection); Resources.Builder resourcesBuilder = Resources.builder(); Datasets updatedDatasets = enrichedDatasets; @@ -201,7 +201,7 @@ private List ingest(Connection connection, Datasets datasets, Li } boolean mainDatasetExists = executor.datasetExists(updatedDatasets.mainDataset()); - if (mainDatasetExists) + if (mainDatasetExists && enableSchemaEvolution()) { updatedDatasets = updatedDatasets.withMainDataset(constructDatasetFromDatabase(executor, updatedDatasets.mainDataset())); } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/executor/RelationalExecutionHelper.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/executor/RelationalExecutionHelper.java new file mode 100644 index 00000000000..d91697cbb01 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/executor/RelationalExecutionHelper.java @@ -0,0 +1,50 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.executor; + +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.sql.JdbcPropertiesToLogicalDataTypeMapping; + +import java.util.List; +import java.util.Map; + +public interface RelationalExecutionHelper +{ + + String COLUMN_NAME = "COLUMN_NAME"; + + void beginTransaction(); + + void commitTransaction(); + + void revertTransaction(); + + void closeTransactionManager(); + + boolean doesTableExist(Dataset dataset); + + void validateDatasetSchema(Dataset dataset, DataTypeMapping datatypeMapping); + + Dataset constructDatasetFromDatabase(String tableName, String schemaName, String databaseName, JdbcPropertiesToLogicalDataTypeMapping mapping); + + void executeStatement(String sql); + + void executeStatements(List sqls); + + List> executeQuery(String sql); + + void close(); +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/executor/RelationalExecutor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/executor/RelationalExecutor.java index 3068c4cc013..62ab267f697 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/executor/RelationalExecutor.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/executor/RelationalExecutor.java @@ -18,7 +18,6 @@ import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; import org.finos.legend.engine.persistence.components.relational.RelationalSink; import org.finos.legend.engine.persistence.components.relational.SqlPlan; -import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcHelper; import org.finos.legend.engine.persistence.components.relational.sql.TabularData; import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; @@ -30,19 +29,19 @@ public class RelationalExecutor implements Executor { private final RelationalSink relationalSink; - private final JdbcHelper jdbcHelper; + private final RelationalExecutionHelper relationalExecutionHelper; - public RelationalExecutor(RelationalSink relationalSink, JdbcHelper jdbcHelper) + public RelationalExecutor(RelationalSink relationalSink, RelationalExecutionHelper relationalExecutionHelper) { this.relationalSink = relationalSink; - this.jdbcHelper = jdbcHelper; + this.relationalExecutionHelper = relationalExecutionHelper; } @Override public void executePhysicalPlan(SqlPlan physicalPlan) { List sqlList = physicalPlan.getSqlList(); - jdbcHelper.executeStatements(sqlList); + relationalExecutionHelper.executeStatements(sqlList); } @Override @@ -52,7 +51,7 @@ public void executePhysicalPlan(SqlPlan physicalPlan, Map placeh for (String sql : sqlList) { String enrichedSql = getEnrichedSql(placeholderKeyValues, sql); - jdbcHelper.executeStatement(enrichedSql); + relationalExecutionHelper.executeStatement(enrichedSql); } } @@ -62,7 +61,7 @@ public List executePhysicalPlanAndGetResults(SqlPlan physicalPlan) List resultSetList = new ArrayList<>(); for (String sql : physicalPlan.getSqlList()) { - List> queryResult = jdbcHelper.executeQuery(sql); + List> queryResult = relationalExecutionHelper.executeQuery(sql); if (!queryResult.isEmpty()) { resultSetList.add(new TabularData(queryResult)); @@ -78,7 +77,7 @@ public List executePhysicalPlanAndGetResults(SqlPlan physicalPlan, for (String sql : physicalPlan.getSqlList()) { String enrichedSql = getEnrichedSql(placeholderKeyValues, sql); - List> queryResult = jdbcHelper.executeQuery(enrichedSql); + List> queryResult = relationalExecutionHelper.executeQuery(enrichedSql); if (!queryResult.isEmpty()) { resultSetList.add(new TabularData(queryResult)); @@ -90,43 +89,43 @@ public List executePhysicalPlanAndGetResults(SqlPlan physicalPlan, @Override public boolean datasetExists(Dataset dataset) { - return relationalSink.datasetExistsFn().apply(this, jdbcHelper, dataset); + return relationalSink.datasetExistsFn().apply(this, relationalExecutionHelper, dataset); } @Override public void validateMainDatasetSchema(Dataset dataset) { - relationalSink.validateMainDatasetSchemaFn().execute(this, jdbcHelper, dataset); + relationalSink.validateMainDatasetSchemaFn().execute(this, relationalExecutionHelper, dataset); } @Override public Dataset constructDatasetFromDatabase(String tableName, String schemaName, String databaseName) { - return relationalSink.constructDatasetFromDatabaseFn().execute(this, jdbcHelper, tableName, schemaName, databaseName); + return relationalSink.constructDatasetFromDatabaseFn().execute(this, relationalExecutionHelper, tableName, schemaName, databaseName); } @Override public void begin() { - jdbcHelper.beginTransaction(); + relationalExecutionHelper.beginTransaction(); } @Override public void commit() { - jdbcHelper.commitTransaction(); + relationalExecutionHelper.commitTransaction(); } @Override public void revert() { - jdbcHelper.revertTransaction(); + relationalExecutionHelper.revertTransaction(); } @Override public void close() { - jdbcHelper.closeTransactionManager(); + relationalExecutionHelper.closeTransactionManager(); } private String getEnrichedSql(Map placeholderKeyValues, String sql) diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcConnectionAbstract.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcConnectionAbstract.java new file mode 100644 index 00000000000..1802a4e32e1 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcConnectionAbstract.java @@ -0,0 +1,34 @@ +// Copyright 2022 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.jdbc; + +import org.finos.legend.engine.persistence.components.relational.api.RelationalConnection; +import org.immutables.value.Value; + +import java.sql.Connection; + +@Value.Immutable +@Value.Style( + typeAbstract = "*Abstract", + typeImmutable = "*", + jdkOnly = true, + optionalAcceptNullable = true, + strictBuilder = true +) +public abstract class JdbcConnectionAbstract implements RelationalConnection +{ + @Value.Parameter(order = 0) + public abstract Connection connection(); +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcHelper.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcHelper.java index bec556114a5..0e8722d4d59 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcHelper.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcHelper.java @@ -20,6 +20,7 @@ import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Index; import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; +import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutionHelper; import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; import org.finos.legend.engine.persistence.components.relational.sql.JdbcPropertiesToLogicalDataTypeMapping; import org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause; @@ -46,14 +47,13 @@ import java.util.Set; import java.util.regex.Pattern; -public class JdbcHelper +public class JdbcHelper implements RelationalExecutionHelper { private static final Logger LOGGER = LoggerFactory.getLogger(JdbcHelper.class); private final Connection connection; private JdbcTransactionManager transactionManager; - public static final String COLUMN_NAME = "COLUMN_NAME"; public static final String TYPE_NAME = "TYPE_NAME"; public static final String DATA_TYPE = "DATA_TYPE"; public static final String COLUMN_SIZE = "COLUMN_SIZE"; @@ -79,6 +79,7 @@ public Connection connection() return connection; } + @Override public void beginTransaction() { try @@ -92,6 +93,7 @@ public void beginTransaction() } } + @Override public void commitTransaction() { if (this.transactionManager != null) @@ -107,6 +109,7 @@ public void commitTransaction() } } + @Override public void revertTransaction() { if (this.transactionManager != null) @@ -122,6 +125,7 @@ public void revertTransaction() } } + @Override public void closeTransactionManager() { try @@ -138,6 +142,7 @@ public void closeTransactionManager() } } + @Override public boolean doesTableExist(Dataset dataset) { try @@ -145,7 +150,7 @@ public boolean doesTableExist(Dataset dataset) String name = dataset.datasetReference().name().orElseThrow(IllegalStateException::new); String database = dataset.datasetReference().database().orElse(null); String schema = dataset.datasetReference().group().orElse(null); - ResultSet result = this.connection.getMetaData().getTables(database, schema, name, new String[]{Clause.TABLE.get()}); + ResultSet result = this.connection.getMetaData().getTables(database, schema, name, new String[] {Clause.TABLE.get()}); return result.isBeforeFirst(); // This method returns true if ResultSet is not empty } catch (SQLException e) @@ -154,6 +159,7 @@ public boolean doesTableExist(Dataset dataset) } } + @Override public void validateDatasetSchema(Dataset dataset, DataTypeMapping datatypeMapping) { try @@ -173,7 +179,7 @@ public void validateDatasetSchema(Dataset dataset, DataTypeMapping datatypeMappi ResultSet primaryKeyResult = dbMetaData.getPrimaryKeys(database, schema, name); while (primaryKeyResult.next()) { - primaryKeys.add(primaryKeyResult.getString(COLUMN_NAME)); + primaryKeys.add(primaryKeyResult.getString(RelationalExecutionHelper.COLUMN_NAME)); } // Get unique keys @@ -181,12 +187,12 @@ public void validateDatasetSchema(Dataset dataset, DataTypeMapping datatypeMappi ResultSet uniqueKeyResult = dbMetaData.getIndexInfo(database, schema, name, true, false); while (uniqueKeyResult.next()) { - uniqueKeys.add(uniqueKeyResult.getString(COLUMN_NAME)); + uniqueKeys.add(uniqueKeyResult.getString(RelationalExecutionHelper.COLUMN_NAME)); } while (result.next()) { - String columnName = result.getString(COLUMN_NAME); + String columnName = result.getString(RelationalExecutionHelper.COLUMN_NAME); // Get the datatype String typeName = result.getString(TYPE_NAME); @@ -243,6 +249,7 @@ public void validateDatasetSchema(Dataset dataset, DataTypeMapping datatypeMappi } } + @Override public Dataset constructDatasetFromDatabase(String tableName, String schemaName, String databaseName, JdbcPropertiesToLogicalDataTypeMapping mapping) { try @@ -254,7 +261,7 @@ public Dataset constructDatasetFromDatabase(String tableName, String schemaName, ResultSet primaryKeyResult = dbMetaData.getPrimaryKeys(databaseName, schemaName, tableName); while (primaryKeyResult.next()) { - primaryKeys.add(primaryKeyResult.getString(COLUMN_NAME)); + primaryKeys.add(primaryKeyResult.getString(RelationalExecutionHelper.COLUMN_NAME)); } // Get all unique constraints and indices @@ -265,7 +272,7 @@ public Dataset constructDatasetFromDatabase(String tableName, String schemaName, while (indexResult.next()) { String indexName = indexResult.getString(INDEX_NAME); - String columnName = indexResult.getString(COLUMN_NAME); + String columnName = indexResult.getString(RelationalExecutionHelper.COLUMN_NAME); boolean isIndexNonUnique = indexResult.getBoolean(NON_UNIQUE); if (!indexName.matches(Pattern.compile("PRIMARY_KEY_[a-zA-Z0-9]+").pattern())) @@ -303,7 +310,7 @@ public Dataset constructDatasetFromDatabase(String tableName, String schemaName, ResultSet columnResult = dbMetaData.getColumns(databaseName, schemaName, tableName, null); while (columnResult.next()) { - String columnName = columnResult.getString(COLUMN_NAME); + String columnName = columnResult.getString(RelationalExecutionHelper.COLUMN_NAME); String typeName = columnResult.getString(TYPE_NAME); String dataType = JDBCType.valueOf(columnResult.getInt(DATA_TYPE)).getName(); int columnSize = columnResult.getInt(COLUMN_SIZE); @@ -379,12 +386,14 @@ public static List convertUserProvidedFieldsToColumns(List userFi } + @Override public void executeStatement(String sql) { List sqls = Collections.singletonList(sql); executeStatements(sqls); } + @Override public void executeStatements(List sqls) { if (this.transactionManager != null) @@ -447,6 +456,7 @@ public void executeStatements(List sqls) } } + @Override public List> executeQuery(String sql) { if (this.transactionManager != null) @@ -482,6 +492,7 @@ public List> executeQuery(String sql) } } + @Override public void close() { try diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcTransactionManager.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcTransactionManager.java index 3e1ef59d8dd..9f2b28cc38f 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcTransactionManager.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcTransactionManager.java @@ -62,7 +62,7 @@ public void revertTransaction() throws SQLException this.setAutoCommitToPreviousState(); } - public void setAutoCommitToPreviousState() throws SQLException + private void setAutoCommitToPreviousState() throws SQLException { this.connection.setAutoCommit(this.previousAutoCommit); } @@ -72,18 +72,6 @@ public boolean executeInCurrentTransaction(String sql) throws SQLException return this.statement.execute(sql); } - public void addToCurrentBatchForExecution(String sql) throws SQLException - { - this.statement.addBatch(sql); - } - - public int[] executeCurrentBatchInTransaction() throws SQLException - { - int[] batchResults = this.statement.executeBatch(); - this.statement.clearBatch(); - return batchResults; - } - // todo: find a better way to return both the data and schema public List> convertResultSetToList(String sql) { diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sql/JdbcPropertiesToLogicalDataTypeMapping.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sql/JdbcPropertiesToLogicalDataTypeMapping.java index ade6ef34398..20193b12ae2 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sql/JdbcPropertiesToLogicalDataTypeMapping.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sql/JdbcPropertiesToLogicalDataTypeMapping.java @@ -44,5 +44,5 @@ public interface JdbcPropertiesToLogicalDataTypeMapping String ARRAY = "ARRAY"; String CHARACTER_VARYING = "CHARACTER VARYING"; - FieldType getDataType(String typeName, String dataType, int columnSize, int decimalDigits); + FieldType getDataType(String typeName, String dataType, Integer columnSize, Integer decimalDigits); } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/common/Clause.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/common/Clause.java index 6eb925eacb8..4fb85be11dd 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/common/Clause.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/common/Clause.java @@ -54,7 +54,9 @@ public enum Clause OVER("OVER"), PARTITION_BY("PARTITION BY"), ORDER_BY("ORDER BY"), - CLUSTER_BY("CLUSTER BY"); + CLUSTER_BY("CLUSTER BY"), + NOT_ENFORCED("NOT ENFORCED"), + DATA_TYPE("DATA TYPE"); private final String clause; diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/common/FunctionName.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/common/FunctionName.java index 46db9493c87..f71dbefefb1 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/common/FunctionName.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/common/FunctionName.java @@ -32,9 +32,17 @@ public enum FunctionName RAW_TO_HEX("RAWTOHEX"), CURRENT_TIMESTAMP("CURRENT_TIMESTAMP"), SYSDATE("SYSDATE"), + CURRENT_DATETIME("CURRENT_DATETIME"), UPPER("UPPER"), SUBSTRING("SUBSTRING"), ROW_NUMBER("ROW_NUMBER"), + DATE("DATE"), + DATE_TRUNC("DATE_TRUNC"), + DATETIME_TRUNC("DATETIME_TRUNC"), + TIMESTAMP_TRUNC("TIMESTAMP_TRUNC"), + RANGE_BUCKET("RANGE_BUCKET"), + GENERATE_ARRAY("GENERATE_ARRAY"), + PARSE_DATETIME("PARSE_DATETIME"), PARSE_JSON("PARSE_JSON"); private static final Map BY_NAME = Arrays diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/constraints/table/PartitionKeyConstraint.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/constraints/table/PartitionKeyConstraint.java new file mode 100644 index 00000000000..9b95adcc6c0 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/constraints/table/PartitionKeyConstraint.java @@ -0,0 +1,39 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.sqldom.constraints.table; + +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlDomException; +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.values.Value; + +public class PartitionKeyConstraint implements SqlGen +{ + private Value key; + + @Override + public void genSql(StringBuilder builder) throws SqlDomException + { + key.genSql(builder); + } + + @Override + public void push(Object node) + { + if (node instanceof Value) + { + key = (Value) node; + } + } +} diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/constraints/table/PrimaryKeyTableConstraint.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/constraints/table/PrimaryKeyTableConstraint.java index 9d759069da0..f98f5d0641e 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/constraints/table/PrimaryKeyTableConstraint.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/constraints/table/PrimaryKeyTableConstraint.java @@ -20,15 +20,27 @@ import java.util.List; import java.util.stream.Collectors; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.NOT_ENFORCED; + public class PrimaryKeyTableConstraint extends TableConstraint { private final List columnNames; private final String quoteIdentifier; + private boolean notEnforced; + public PrimaryKeyTableConstraint(List columnNames, String quoteIdentifier) { this.columnNames = columnNames; this.quoteIdentifier = quoteIdentifier; + this.notEnforced = false; + } + + public PrimaryKeyTableConstraint(List columnNames, String quoteIdentifier, boolean notEnforced) + { + this.columnNames = columnNames; + this.quoteIdentifier = quoteIdentifier; + this.notEnforced = notEnforced; } public List getColumnNames() @@ -38,7 +50,7 @@ public List getColumnNames() public PrimaryKeyTableConstraint withColumnNames(List columnNames) { - return new PrimaryKeyTableConstraint(columnNames, quoteIdentifier); + return new PrimaryKeyTableConstraint(columnNames, quoteIdentifier, notEnforced); } @Override @@ -47,6 +59,10 @@ public void genSql(StringBuilder builder) throws SqlDomException validate(); String primaryKeys = columnNames.stream().map(column -> SqlGenUtils.getQuotedField(column, quoteIdentifier)).collect(Collectors.joining(SqlGenUtils.COMMA + SqlGenUtils.WHITE_SPACE)); builder.append(String.format("PRIMARY KEY (%s)", primaryKeys)); + if (notEnforced) + { + builder.append(" " + NOT_ENFORCED.get()); + } } @Override diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/Column.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/Column.java index d87a41922dc..9ac3de40832 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/Column.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/Column.java @@ -82,6 +82,12 @@ public void genSqlWithNameAndTypeOnly(StringBuilder builder) throws SqlDomExcept dataType.genSql(builder); } + public void genSqlWithTypeOnly(StringBuilder builder) throws SqlDomException + { + validate(); + dataType.genSql(builder); + } + @Override public void push(Object node) { diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/statements/InsertStatement.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/statements/InsertStatement.java index df873c47e15..c1addb84550 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/statements/InsertStatement.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/statements/InsertStatement.java @@ -17,6 +17,7 @@ import org.finos.legend.engine.persistence.components.relational.sqldom.SqlDomException; import org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause; import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.expresssions.select.SelectExpression; +import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.expresssions.select.ValuesExpression; import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.expresssions.table.Table; import org.finos.legend.engine.persistence.components.relational.sqldom.schemaops.values.Field; @@ -81,9 +82,16 @@ public void genSql(StringBuilder builder) throws SqlDomException } builder.append(WHITE_SPACE); - builder.append(OPEN_PARENTHESIS); - selectExpression.genSql(builder); - builder.append(CLOSING_PARENTHESIS); + if (selectExpression instanceof ValuesExpression) + { + selectExpression.genSql(builder); + } + else + { + builder.append(OPEN_PARENTHESIS); + selectExpression.genSql(builder); + builder.append(CLOSING_PARENTHESIS); + } } @Override diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/values/WindowFunction.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/values/WindowFunction.java index 05f3b4dfe09..243caae8648 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/values/WindowFunction.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/values/WindowFunction.java @@ -20,8 +20,13 @@ import java.util.ArrayList; import java.util.List; -import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.*; -import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.*; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.ORDER_BY; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.OVER; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.PARTITION_BY; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.CLOSING_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.COMMA; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.OPEN_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.WHITE_SPACE; public class WindowFunction extends Value { diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/test/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/InsertTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/test/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/InsertTest.java index d7af328890c..706cbd9e2f9 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/test/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/InsertTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/test/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/InsertTest.java @@ -56,7 +56,7 @@ void testInsertWithValues() String sql = BaseTest.genSqlIgnoringErrors(insertStatement); - assertEquals("INSERT INTO \"mydb\".\"mytable\" (\"col1\", \"col2\", \"col3\", \"col4\") (VALUES (1,'2',3.05,4),(11,'22',33.05,44))", sql); + assertEquals("INSERT INTO \"mydb\".\"mytable\" (\"col1\", \"col2\", \"col3\", \"col4\") VALUES (1,'2',3.05,4),(11,'22',33.05,44)", sql); } @Test diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/H2Sink.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/H2Sink.java index 4e23a96477a..76f7deb7726 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/H2Sink.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/H2Sink.java @@ -15,8 +15,11 @@ package org.finos.legend.engine.persistence.components.relational.h2; import java.util.Optional; + +import org.finos.legend.engine.persistence.components.executor.Executor; import org.finos.legend.engine.persistence.components.logicalplan.datasets.CsvExternalDatasetReference; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; import org.finos.legend.engine.persistence.components.logicalplan.operations.LoadCsv; import org.finos.legend.engine.persistence.components.logicalplan.values.HashFunction; @@ -24,6 +27,7 @@ import org.finos.legend.engine.persistence.components.optimizer.Optimizer; import org.finos.legend.engine.persistence.components.relational.CaseConversion; import org.finos.legend.engine.persistence.components.relational.RelationalSink; +import org.finos.legend.engine.persistence.components.relational.SqlPlan; import org.finos.legend.engine.persistence.components.relational.ansi.AnsiSqlSink; import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.LowerCaseOptimizer; import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.UpperCaseOptimizer; @@ -34,9 +38,16 @@ import org.finos.legend.engine.persistence.components.relational.h2.sql.visitor.LoadCsvVisitor; import org.finos.legend.engine.persistence.components.relational.h2.sql.visitor.SchemaDefinitionVisitor; import org.finos.legend.engine.persistence.components.relational.h2.sql.visitor.ParseJsonFunctionVisitor; +import org.finos.legend.engine.persistence.components.relational.h2.sql.visitor.FieldVisitor; import org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils; import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; import org.finos.legend.engine.persistence.components.util.Capability; +import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcConnection; +import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcHelper; +import org.finos.legend.engine.persistence.components.relational.sql.TabularData; +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; +import org.finos.legend.engine.persistence.components.relational.api.RelationalConnection; +import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutor; import java.sql.Connection; import java.sql.DriverManager; @@ -74,6 +85,7 @@ public class H2Sink extends AnsiSqlSink logicalPlanVisitorByClass.put(ParseJsonFunction.class, new ParseJsonFunctionVisitor()); logicalPlanVisitorByClass.put(LoadCsv.class, new LoadCsvVisitor()); logicalPlanVisitorByClass.put(CsvExternalDatasetReference.class, new CsvExternalDatasetReferenceVisitor()); + logicalPlanVisitorByClass.put(Field.class, new FieldVisitor()); LOGICAL_PLAN_VISITOR_BY_CLASS = Collections.unmodifiableMap(logicalPlanVisitorByClass); Map> implicitDataTypeMapping = new HashMap<>(); @@ -131,6 +143,20 @@ private H2Sink() (executor, sink, tableName, schemaName, databaseName) -> sink.constructDatasetFromDatabase(tableName, schemaName, databaseName, new H2JdbcPropertiesToLogicalDataTypeMapping())); } + @Override + public Executor getRelationalExecutor(RelationalConnection relationalConnection) + { + if (relationalConnection instanceof JdbcConnection) + { + JdbcConnection jdbcConnection = (JdbcConnection) relationalConnection; + return new RelationalExecutor(this, JdbcHelper.of(jdbcConnection.connection())); + } + else + { + throw new UnsupportedOperationException("Only JdbcConnection is supported for H2 Sink"); + } + } + @Override public Optional optimizerForCaseConversion(CaseConversion caseConversion) { diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/H2JdbcPropertiesToLogicalDataTypeMapping.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/H2JdbcPropertiesToLogicalDataTypeMapping.java index ff3043a5d4a..d106e4d3ead 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/H2JdbcPropertiesToLogicalDataTypeMapping.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/H2JdbcPropertiesToLogicalDataTypeMapping.java @@ -20,7 +20,7 @@ public class H2JdbcPropertiesToLogicalDataTypeMapping implements JdbcPropertiesToLogicalDataTypeMapping { - public FieldType getDataType(String typeName, String dataType, int columnSize, int decimalDigits) + public FieldType getDataType(String typeName, String dataType, Integer columnSize, Integer decimalDigits) { switch (typeName) diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/visitor/FieldVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/visitor/FieldVisitor.java new file mode 100644 index 00000000000..e03371b80c3 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/main/java/org/finos/legend/engine/persistence/components/relational/h2/sql/visitor/FieldVisitor.java @@ -0,0 +1,26 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.h2.sql.visitor; + +import org.finos.legend.engine.persistence.components.relational.h2.sql.H2DataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; + +public class FieldVisitor extends org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.FieldVisitor +{ + public DataTypeMapping getDataTypeMapping() + { + return new H2DataTypeMapping(); + } +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/BaseTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/BaseTest.java index 5964153b71b..8adf67c5ef0 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/BaseTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/BaseTest.java @@ -29,6 +29,7 @@ import org.finos.legend.engine.persistence.components.relational.api.RelationalIngestor; import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutor; import org.finos.legend.engine.persistence.components.relational.h2.H2Sink; +import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcConnection; import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcHelper; import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; import org.finos.legend.engine.persistence.components.util.SchemaEvolutionCapability; @@ -145,7 +146,7 @@ protected IngestorResult executePlansAndVerifyResults(IngestMode ingestMode, Pla .enableSchemaEvolution(options.enableSchemaEvolution()) .schemaEvolutionCapabilitySet(userCapabilitySet) .build(); - IngestorResult result = ingestor.ingest(h2Sink.connection(), datasets); + IngestorResult result = ingestor.ingest(JdbcConnection.of(h2Sink.connection()), datasets); Map actualStats = result.statisticByName(); @@ -185,7 +186,7 @@ protected List executePlansAndVerifyResultsWithDataSplits(Ingest .enableSchemaEvolution(options.enableSchemaEvolution()) .build(); - List results = ingestor.ingestWithDataSplits(h2Sink.connection(), datasets, dataSplitRanges); + List results = ingestor.ingestWithDataSplits(JdbcConnection.of(h2Sink.connection()), datasets, dataSplitRanges); List> tableData = h2Sink.executeQuery("select * from \"TEST\".\"main\""); TestUtils.assertFileAndTableDataEquals(schema, expectedDataPath, tableData); @@ -231,7 +232,7 @@ public IngestorResult executePlansAndVerifyForCaseConversion(IngestMode ingestMo .caseConversion(CaseConversion.TO_UPPER) .build(); - IngestorResult result = ingestor.ingest(h2Sink.connection(), datasets); + IngestorResult result = ingestor.ingest(JdbcConnection.of(h2Sink.connection()), datasets); Map actualStats = result.statisticByName(); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/TestUtils.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/TestUtils.java index 64278bca2a1..a29808ae922 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/TestUtils.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/TestUtils.java @@ -26,7 +26,7 @@ import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; import org.finos.legend.engine.persistence.components.logicalplan.datasets.JsonExternalDatasetReference; import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; -import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcHelper; +import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutionHelper; import org.finos.legend.engine.persistence.components.util.MetadataDataset; import org.junit.jupiter.api.Assertions; @@ -1154,14 +1154,14 @@ public static void assertTableColumnsEquals(List expectedSchema, List> result = sink.executeQuery("SELECT IS_NULLABLE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='" + tableName + "' and COLUMN_NAME ='" + columnName + "'"); return result.get(0).get("IS_NULLABLE").toString(); } // This is to check the actual database table - the length (precision) of the column data type - public static int getColumnDataTypeLengthFromTable(JdbcHelper sink, String tableName, String columnName) + public static int getColumnDataTypeLengthFromTable(RelationalExecutionHelper sink, String tableName, String columnName) { List> result = sink.executeQuery("SELECT NUMERIC_PRECISION, CHARACTER_MAXIMUM_LENGTH FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='" + tableName + "' and COLUMN_NAME ='" + columnName + "'"); Object precisionOrLength = Optional.ofNullable(result.get(0).get("NUMERIC_PRECISION")).orElseGet(() -> @@ -1171,7 +1171,7 @@ public static int getColumnDataTypeLengthFromTable(JdbcHelper sink, String table } // This is to check the actual database table - the scale of the column data type - public static int getColumnDataTypeScaleFromTable(JdbcHelper sink, String tableName, String columnName) + public static int getColumnDataTypeScaleFromTable(RelationalExecutionHelper sink, String tableName, String columnName) { List> result = sink.executeQuery("SELECT NUMERIC_SCALE FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME='" + tableName + "' and COLUMN_NAME ='" + columnName + "'"); return Integer.parseInt(result.get(0).get("NUMERIC_SCALE").toString()); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcHelperTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcHelperTest.java index 36ae033823b..3271a4819c0 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcHelperTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-h2/src/test/java/org/finos/legend/engine/persistence/components/relational/jdbc/JdbcHelperTest.java @@ -14,6 +14,7 @@ package org.finos.legend.engine.persistence.components.relational.jdbc; +import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutionHelper; import org.finos.legend.engine.persistence.components.relational.h2.H2Sink; import org.junit.jupiter.api.Test; @@ -35,7 +36,7 @@ public class JdbcHelperTest @Test void prepare() throws Exception { - JdbcHelper sink = JdbcHelper.of(H2Sink.createConnection(H2_USER_NAME, H2_PASSWORD, H2_JDBC_URL)); + RelationalExecutionHelper sink = JdbcHelper.of(H2Sink.createConnection(H2_USER_NAME, H2_PASSWORD, H2_JDBC_URL)); // Create table example List list2 = new ArrayList<>(); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/MemSqlSink.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/MemSqlSink.java index f8c14953956..9eb35b96bbf 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/MemSqlSink.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/MemSqlSink.java @@ -33,6 +33,10 @@ import org.finos.legend.engine.persistence.components.relational.ansi.AnsiSqlSink; import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.LowerCaseOptimizer; import org.finos.legend.engine.persistence.components.relational.ansi.optimizer.UpperCaseOptimizer; +import org.finos.legend.engine.persistence.components.relational.api.RelationalConnection; +import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutionHelper; +import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutor; +import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcConnection; import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcHelper; import org.finos.legend.engine.persistence.components.relational.memsql.sql.MemSqlDataTypeMapping; import org.finos.legend.engine.persistence.components.relational.memsql.sql.visitor.AlterVisitor; @@ -40,6 +44,7 @@ import org.finos.legend.engine.persistence.components.relational.memsql.sql.visitor.SchemaDefinitionVisitor; import org.finos.legend.engine.persistence.components.relational.memsql.sql.visitor.ShowVisitor; import org.finos.legend.engine.persistence.components.relational.memsql.sql.visitor.SQLCreateVisitor; +import org.finos.legend.engine.persistence.components.relational.memsql.sql.visitor.FieldVisitor; import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; import org.finos.legend.engine.persistence.components.relational.sql.TabularData; import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; @@ -91,6 +96,7 @@ public class MemSqlSink extends AnsiSqlSink logicalPlanVisitorByClass.put(Create.class, new SQLCreateVisitor()); logicalPlanVisitorByClass.put(Show.class, new ShowVisitor()); logicalPlanVisitorByClass.put(Update.class, new SQLUpdateVisitor()); + logicalPlanVisitorByClass.put(Field.class, new FieldVisitor()); LOGICAL_PLAN_VISITOR_BY_CLASS = Collections.unmodifiableMap(logicalPlanVisitorByClass); Map> implicitDataTypeMapping = new HashMap<>(); @@ -156,6 +162,20 @@ private MemSqlSink() }); } + @Override + public Executor getRelationalExecutor(RelationalConnection relationalConnection) + { + if (relationalConnection instanceof JdbcConnection) + { + JdbcConnection jdbcConnection = (JdbcConnection) relationalConnection; + return new RelationalExecutor(this, JdbcHelper.of(jdbcConnection.connection())); + } + else + { + throw new UnsupportedOperationException("Only JdbcConnection is supported for MemSql Sink"); + } + } + @Override public Optional optimizerForCaseConversion(CaseConversion caseConversion) @@ -176,7 +196,7 @@ public Optional optimizerForCaseConversion(CaseConversion caseConvers static final ValidateMainDatasetSchema VALIDATE_MAIN_DATASET_SCHEMA = new ValidateMainDatasetSchema() { @Override - public void execute(Executor executor, JdbcHelper sink, Dataset dataset) + public void execute(Executor executor, RelationalExecutionHelper sink, Dataset dataset) { RelationalTransformer transformer = new RelationalTransformer(MemSqlSink.get()); LogicalPlan validateDatasetSchemaLogicalPlan = LogicalPlanFactory.getLogicalPlanForValidateDatasetSchema(dataset); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/sql/visitor/FieldVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/sql/visitor/FieldVisitor.java new file mode 100644 index 00000000000..d966bdef976 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/sql/visitor/FieldVisitor.java @@ -0,0 +1,26 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.memsql.sql.visitor; + +import org.finos.legend.engine.persistence.components.relational.memsql.sql.MemSqlDataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; + +public class FieldVisitor extends org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.FieldVisitor +{ + public DataTypeMapping getDataTypeMapping() + { + return new MemSqlDataTypeMapping(); + } +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/sql/visitor/SQLCreateVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/sql/visitor/SQLCreateVisitor.java index 814d8f42cd1..cbeaa125245 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/sql/visitor/SQLCreateVisitor.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/main/java/org/finos/legend/engine/persistence/components/relational/memsql/sql/visitor/SQLCreateVisitor.java @@ -43,7 +43,7 @@ public VisitorResult visit(PhysicalPlanNode prev, Create current, VisitorContext prev.push(createTable); List logicalPlanNodes = new ArrayList<>(); - logicalPlanNodes.add(current.dataset()); + logicalPlanNodes.add(current.dataset().datasetReference()); logicalPlanNodes.add(current.dataset().schema()); if (current.ifNotExists()) diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaTest.java index d70fba84640..ea5cdc88bb3 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaTest.java @@ -269,9 +269,11 @@ public void verifyNontemporalDeltaWithNoVersionAndStagingFilter(GeneratorResult "ON ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` <> stage.`digest`) " + "SET sink.`id` = stage.`id`,sink.`name` = stage.`name`,sink.`amount` = stage.`amount`,sink.`biz_date` = stage.`biz_date`,sink.`digest` = stage.`digest`"; - String insertSql = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`, `digest`) " + - "(SELECT * FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest` FROM `mydb`.`staging` as stage WHERE (stage.`biz_date` > '2020-01-01') AND (stage.`biz_date` < '2020-01-03')) as stage " + - "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))"; + String insertSql = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`) " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`main` as sink WHERE (sink.`id` = stage.`id`) AND " + + "(sink.`name` = stage.`name`)))) AND ((stage.`biz_date` > '2020-01-01') AND (stage.`biz_date` < '2020-01-03')))"; Assertions.assertEquals(MemsqlTestArtifacts.expectedBaseTablePlusDigestCreateQuery, preActionsSqlList.get(0)); Assertions.assertEquals(updateSql, milestoningSqlList.get(0)); @@ -327,9 +329,11 @@ public void verifyNontemporalDeltaWithMaxVersioningNoDedupAndStagingFilters(Gene "ON ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (stage.`version` > sink.`version`) " + "SET sink.`id` = stage.`id`,sink.`name` = stage.`name`,sink.`amount` = stage.`amount`,sink.`biz_date` = stage.`biz_date`,sink.`digest` = stage.`digest`,sink.`version` = stage.`version`"; - String insertSql = "INSERT INTO `mydb`.`main` (`id`, `name`, `amount`, `biz_date`, `digest`, `version`) " + - "(SELECT * FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version` FROM `mydb`.`staging` as stage WHERE stage.`snapshot_id` > 18972) as stage " + - "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))"; + String insertSql = "INSERT INTO `mydb`.`main` " + + "(`id`, `name`, `amount`, `biz_date`, `digest`, `version`) " + + "(SELECT * FROM `mydb`.`staging` as stage WHERE (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`main` as sink WHERE (sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))) " + + "AND (stage.`snapshot_id` > 18972))"; Assertions.assertEquals(MemsqlTestArtifacts.expectedBaseTablePlusDigestPlusVersionCreateQuery, preActionsSqlList.get(0)); Assertions.assertEquals(updateSql, milestoningSqlList.get(0)); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdBasedTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdBasedTest.java index 935cf978306..fa0e87c2973 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdBasedTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-memsql/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/UnitemporalDeltaBatchIdBasedTest.java @@ -293,22 +293,20 @@ public void verifyUnitemporalDeltaWithNoVersionAndStagingFilter(GeneratorResult List preActionsSql = operations.preActionsSql(); List milestoningSql = operations.ingestSql(); List metadataIngestSql = operations.metadataIngestSql(); - String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + - "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + - "WHERE (sink.`batch_id_out` = 999999999) AND " + - "(EXISTS (SELECT * FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest` FROM `mydb`.`staging` as stage WHERE stage.`batch_id_in` > 5) as stage " + - "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + - "(sink.`digest` <> stage.`digest`)))"; + String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink SET sink.`batch_id_out` = " + + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + + "WHERE (sink.`batch_id_out` = 999999999) AND (EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE (((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND (sink.`digest` <> stage.`digest`)) " + + "AND (stage.`batch_id_in` > 5)))"; String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + "(`id`, `name`, `amount`, `biz_date`, `digest`, `batch_id_in`, `batch_id_out`) " + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`," + - "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + - "999999999 " + - "FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest` FROM `mydb`.`staging` as stage WHERE stage.`batch_id_in` > 5) as stage " + - "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + - "WHERE (sink.`batch_id_out` = 999999999) " + - "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE " + + "UPPER(batch_metadata.`table_name`) = 'MAIN'),999999999 FROM `mydb`.`staging` as stage " + + "WHERE (NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) " + + "AND (sink.`digest` = stage.`digest`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) " + + "AND (stage.`batch_id_in` > 5))"; Assertions.assertEquals(MemsqlTestArtifacts.expectedMainTableBatchIdBasedCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(MemsqlTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); @@ -365,20 +363,18 @@ public void verifyUnitemporalDeltaWithMaxVersionNoDedupAndStagingFilter(Generato List metadataIngestSql = operations.metadataIngestSql(); String expectedMilestoneQuery = "UPDATE `mydb`.`main` as sink " + "SET sink.`batch_id_out` = (SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')-1 " + - "WHERE (sink.`batch_id_out` = 999999999) AND " + - "(EXISTS (SELECT * FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version` FROM `mydb`.`staging` as stage WHERE stage.`batch_id_in` > 5) as stage " + - "WHERE ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + - "(stage.`version` > sink.`version`)))"; + "WHERE (sink.`batch_id_out` = 999999999) AND (EXISTS (SELECT * FROM `mydb`.`staging` as stage " + + "WHERE (((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)) AND " + + "(stage.`version` > sink.`version`)) AND (stage.`batch_id_in` > 5)))"; String expectedUpsertQuery = "INSERT INTO `mydb`.`main` " + "(`id`, `name`, `amount`, `biz_date`, `digest`, `version`, `batch_id_in`, `batch_id_out`) " + "(SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version`," + "(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN')," + - "999999999 " + - "FROM (SELECT stage.`id`,stage.`name`,stage.`amount`,stage.`biz_date`,stage.`digest`,stage.`version` FROM `mydb`.`staging` as stage WHERE stage.`batch_id_in` > 5) as stage " + - "WHERE NOT (EXISTS (SELECT * FROM `mydb`.`main` as sink " + - "WHERE (sink.`batch_id_out` = 999999999) " + - "AND (stage.`version` <= sink.`version`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`)))))"; + "999999999 FROM `mydb`.`staging` as stage WHERE (NOT (EXISTS " + + "(SELECT * FROM `mydb`.`main` as sink WHERE (sink.`batch_id_out` = 999999999) AND " + + "(stage.`version` <= sink.`version`) AND ((sink.`id` = stage.`id`) AND (sink.`name` = stage.`name`))))) " + + "AND (stage.`batch_id_in` > 5))"; Assertions.assertEquals(MemsqlTestArtifacts.expectedMainTableBatchIdAndVersionBasedCreateQuery, preActionsSql.get(0)); Assertions.assertEquals(MemsqlTestArtifacts.expectedMetadataTableCreateQuery, preActionsSql.get(1)); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/SnowflakeSink.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/SnowflakeSink.java index af42b020d17..b590dea464b 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/SnowflakeSink.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/SnowflakeSink.java @@ -14,10 +14,12 @@ package org.finos.legend.engine.persistence.components.relational.snowflake; +import org.finos.legend.engine.persistence.components.executor.Executor; import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlan; import org.finos.legend.engine.persistence.components.logicalplan.LogicalPlanFactory; import org.finos.legend.engine.persistence.components.logicalplan.datasets.ClusterKey; import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; import org.finos.legend.engine.persistence.components.logicalplan.operations.Alter; import org.finos.legend.engine.persistence.components.logicalplan.operations.Create; @@ -28,6 +30,10 @@ import org.finos.legend.engine.persistence.components.relational.RelationalSink; import org.finos.legend.engine.persistence.components.relational.SqlPlan; import org.finos.legend.engine.persistence.components.relational.ansi.AnsiSqlSink; +import org.finos.legend.engine.persistence.components.relational.api.RelationalConnection; +import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutor; +import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcConnection; +import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcHelper; import org.finos.legend.engine.persistence.components.relational.snowflake.optmizer.LowerCaseOptimizer; import org.finos.legend.engine.persistence.components.relational.snowflake.optmizer.UpperCaseOptimizer; import org.finos.legend.engine.persistence.components.relational.snowflake.sql.SnowflakeDataTypeMapping; @@ -37,8 +43,10 @@ import org.finos.legend.engine.persistence.components.relational.snowflake.sql.visitor.ClusterKeyVisitor; import org.finos.legend.engine.persistence.components.relational.snowflake.sql.visitor.SQLCreateVisitor; import org.finos.legend.engine.persistence.components.relational.snowflake.sql.visitor.SchemaDefinitionVisitor; +import org.finos.legend.engine.persistence.components.relational.snowflake.sql.visitor.FieldVisitor; import org.finos.legend.engine.persistence.components.relational.snowflake.sql.visitor.ShowVisitor; import org.finos.legend.engine.persistence.components.relational.sql.TabularData; +import org.finos.legend.engine.persistence.components.relational.sqldom.SqlGen; import org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils; import org.finos.legend.engine.persistence.components.relational.transformer.RelationalTransformer; import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; @@ -82,6 +90,8 @@ public class SnowflakeSink extends AnsiSqlSink logicalPlanVisitorByClass.put(Alter.class, new AlterVisitor()); logicalPlanVisitorByClass.put(Show.class, new ShowVisitor()); logicalPlanVisitorByClass.put(BatchEndTimestamp.class, new BatchEndTimestampVisitor()); + logicalPlanVisitorByClass.put(Field.class, new FieldVisitor()); + LOGICAL_PLAN_VISITOR_BY_CLASS = Collections.unmodifiableMap(logicalPlanVisitorByClass); Map> implicitDataTypeMapping = new HashMap<>(); @@ -147,6 +157,20 @@ private SnowflakeSink() (executor, sink, tableName, schemaName, databaseName) -> sink.constructDatasetFromDatabase(tableName, schemaName, databaseName, new SnowflakeJdbcPropertiesToLogicalDataTypeMapping())); } + @Override + public Executor getRelationalExecutor(RelationalConnection relationalConnection) + { + if (relationalConnection instanceof JdbcConnection) + { + JdbcConnection jdbcConnection = (JdbcConnection) relationalConnection; + return new RelationalExecutor(this, JdbcHelper.of(jdbcConnection.connection())); + } + else + { + throw new UnsupportedOperationException("Only JdbcConnection is supported for Snowflake Sink"); + } + } + @Override public Optional optimizerForCaseConversion(CaseConversion caseConversion) { diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/SnowflakeJdbcPropertiesToLogicalDataTypeMapping.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/SnowflakeJdbcPropertiesToLogicalDataTypeMapping.java index c21588e6b48..454553cbfa4 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/SnowflakeJdbcPropertiesToLogicalDataTypeMapping.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/SnowflakeJdbcPropertiesToLogicalDataTypeMapping.java @@ -23,7 +23,7 @@ public class SnowflakeJdbcPropertiesToLogicalDataTypeMapping implements JdbcProp // Reference: https://docs.snowflake.com/en/sql-reference/data-types.html - public FieldType getDataType(String typeName, String dataType, int columnSize, int decimalDigits) + public FieldType getDataType(String typeName, String dataType, Integer columnSize, Integer decimalDigits) { switch (typeName) { diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/FieldVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/FieldVisitor.java new file mode 100644 index 00000000000..795ab06cc36 --- /dev/null +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/FieldVisitor.java @@ -0,0 +1,26 @@ +// Copyright 2023 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.persistence.components.relational.snowflake.sql.visitor; + +import org.finos.legend.engine.persistence.components.relational.snowflake.sql.SnowflakeDataTypeMapping; +import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; + +public class FieldVisitor extends org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.FieldVisitor +{ + public DataTypeMapping getDataTypeMapping() + { + return new SnowflakeDataTypeMapping(); + } +} \ No newline at end of file diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/SQLCreateVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/SQLCreateVisitor.java index 594c6e9d3d7..ecbaf12bf5a 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/SQLCreateVisitor.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/main/java/org/finos/legend/engine/persistence/components/relational/snowflake/sql/visitor/SQLCreateVisitor.java @@ -35,7 +35,7 @@ public VisitorResult visit(PhysicalPlanNode prev, Create current, VisitorContext prev.push(createTable); List logicalPlanNodes = new ArrayList<>(); - logicalPlanNodes.add(current.dataset()); + logicalPlanNodes.add(current.dataset().datasetReference()); logicalPlanNodes.add(current.dataset().schema()); if (current.ifNotExists()) diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaMergeTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaMergeTest.java index 87248650808..c4dfa1958ac 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaMergeTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-snowflake/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/NontemporalDeltaMergeTest.java @@ -78,7 +78,8 @@ public void verifyNontemporalDeltaWithAuditingNoDataSplit(GeneratorResult operat "sink.\"name\" = stage.\"name\"," + "sink.\"amount\" = stage.\"amount\"," + "sink.\"biz_date\" = stage.\"biz_date\"," + - "sink.\"digest\" = stage.\"digest\" " + + "sink.\"digest\" = stage.\"digest\"," + + "sink.\"batch_update_time\" = '2000-01-01 00:00:00' " + "WHEN NOT MATCHED THEN INSERT " + "(\"id\", \"name\", \"amount\", \"biz_date\", \"digest\", \"batch_update_time\") " + "VALUES (stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\",'2000-01-01 00:00:00')"; @@ -124,7 +125,7 @@ public void verifyNonTemporalDeltaWithWithAuditingWithDataSplit(List= '{DATA_SPLIT_LOWER_BOUND_PLACEHOLDER}') AND (stage.\"data_split\" <= '{DATA_SPLIT_UPPER_BOUND_PLACEHOLDER}')) " + "as stage ON (sink.\"id\" = stage.\"id\") AND (sink.\"name\" = stage.\"name\") " + "WHEN MATCHED AND sink.\"digest\" <> stage.\"digest\" " + - "THEN UPDATE SET sink.\"id\" = stage.\"id\",sink.\"name\" = stage.\"name\",sink.\"amount\" = stage.\"amount\",sink.\"biz_date\" = stage.\"biz_date\",sink.\"digest\" = stage.\"digest\" " + + "THEN UPDATE SET sink.\"id\" = stage.\"id\",sink.\"name\" = stage.\"name\",sink.\"amount\" = stage.\"amount\",sink.\"biz_date\" = stage.\"biz_date\",sink.\"digest\" = stage.\"digest\",sink.\"batch_update_time\" = '2000-01-01 00:00:00' " + "WHEN NOT MATCHED " + "THEN INSERT (\"id\", \"name\", \"amount\", \"biz_date\", \"digest\", \"batch_update_time\") " + "VALUES (stage.\"id\",stage.\"name\",stage.\"amount\",stage.\"biz_date\",stage.\"digest\",'2000-01-01 00:00:00')"; diff --git a/legend-engine-xt-persistence-component/pom.xml b/legend-engine-xt-persistence-component/pom.xml index 1a631bbc3a0..40b283e4638 100644 --- a/legend-engine-xt-persistence-component/pom.xml +++ b/legend-engine-xt-persistence-component/pom.xml @@ -35,6 +35,7 @@ legend-engine-xt-persistence-component-relational-h2 legend-engine-xt-persistence-component-relational-memsql legend-engine-xt-persistence-component-relational-snowflake + legend-engine-xt-persistence-component-relational-bigquery diff --git a/legend-engine-xt-persistence-test-runner/src/main/java/org/finos/legend/engine/testable/persistence/extension/PersistenceTestRunner.java b/legend-engine-xt-persistence-test-runner/src/main/java/org/finos/legend/engine/testable/persistence/extension/PersistenceTestRunner.java index a40070a3028..d0a4e283138 100644 --- a/legend-engine-xt-persistence-test-runner/src/main/java/org/finos/legend/engine/testable/persistence/extension/PersistenceTestRunner.java +++ b/legend-engine-xt-persistence-test-runner/src/main/java/org/finos/legend/engine/testable/persistence/extension/PersistenceTestRunner.java @@ -23,6 +23,7 @@ import org.finos.legend.engine.persistence.components.relational.api.IngestorResult; import org.finos.legend.engine.persistence.components.relational.api.RelationalIngestor; import org.finos.legend.engine.persistence.components.relational.h2.H2Sink; +import org.finos.legend.engine.persistence.components.relational.jdbc.JdbcConnection; import org.finos.legend.engine.plan.execution.PlanExecutor; import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; import org.finos.legend.engine.protocol.pure.v1.model.data.ExternalFormatData; @@ -152,7 +153,7 @@ private IngestorResult invokePersistence(Dataset targetDataset, Persistence pers .enableSchemaEvolution(SCHEMA_EVOLUTION_DEFAULT) .build(); - IngestorResult result = ingestor.ingest(connection, enrichedDatasets); + IngestorResult result = ingestor.ingest(JdbcConnection.of(connection), enrichedDatasets); return result; }