diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/DeriveMainDatasetSchemaFromStaging.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/DeriveMainDatasetSchemaFromStaging.java index 0187769c761..293daac0e41 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/DeriveMainDatasetSchemaFromStaging.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/DeriveMainDatasetSchemaFromStaging.java @@ -33,7 +33,12 @@ import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.derivation.SourceSpecifiesFromAndThruDateTimeAbstract; import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.derivation.SourceSpecifiesFromDateTimeAbstract; import org.finos.legend.engine.persistence.components.ingestmode.validitymilestoning.derivation.ValidityDerivationVisitor; -import org.finos.legend.engine.persistence.components.logicalplan.datasets.*; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DataType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; import java.util.ArrayList; import java.util.List; diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeOptimizationColumnHandler.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeOptimizationColumnHandler.java index 4da6689ef59..151822fb31e 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeOptimizationColumnHandler.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeOptimizationColumnHandler.java @@ -22,7 +22,8 @@ import java.util.List; import java.util.stream.Collectors; -import static org.finos.legend.engine.persistence.components.util.LogicalPlanUtils.*; +import static org.finos.legend.engine.persistence.components.util.LogicalPlanUtils.SUPPORTED_DATA_TYPES_FOR_OPTIMIZATION_COLUMNS; +import static org.finos.legend.engine.persistence.components.util.LogicalPlanUtils.findCommonPrimaryFieldsBetweenMainAndStaging; public class IngestModeOptimizationColumnHandler implements IngestModeVisitor { @@ -92,9 +93,9 @@ private List deriveOptimizationFilters(UnitemporalDeltaAbstr List primaryKeys = findCommonPrimaryFieldsBetweenMainAndStaging(datasets.mainDataset(), datasets.stagingDataset()); List comparablePrimaryKeys = primaryKeys.stream().filter(field -> SUPPORTED_DATA_TYPES_FOR_OPTIMIZATION_COLUMNS.contains(field.type().dataType())).collect(Collectors.toList()); optimizationFilters = new ArrayList<>(); - for (Field field: comparablePrimaryKeys) + for (Field field : comparablePrimaryKeys) { - OptimizationFilter filter = OptimizationFilter.of(field.name(),field.name().toUpperCase() + "_LOWER", field.name().toUpperCase() + "_UPPER"); + OptimizationFilter filter = OptimizationFilter.of(field.name(), field.name().toUpperCase() + "_LOWER", field.name().toUpperCase() + "_UPPER"); optimizationFilters.add(filter); } } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetDeduplicator.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetDeduplicator.java index 852df86828c..275515bb448 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetDeduplicator.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-logical-plan/src/main/java/org/finos/legend/engine/persistence/components/ingestmode/deduplication/DatasetDeduplicator.java @@ -18,7 +18,14 @@ import org.finos.legend.engine.persistence.components.logicalplan.conditions.Equals; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Selection; -import org.finos.legend.engine.persistence.components.logicalplan.values.*; +import org.finos.legend.engine.persistence.components.logicalplan.values.FieldValue; +import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionImpl; +import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionName; +import org.finos.legend.engine.persistence.components.logicalplan.values.ObjectValue; +import org.finos.legend.engine.persistence.components.logicalplan.values.Order; +import org.finos.legend.engine.persistence.components.logicalplan.values.OrderedField; +import org.finos.legend.engine.persistence.components.logicalplan.values.Value; +import org.finos.legend.engine.persistence.components.logicalplan.values.WindowFunction; import java.util.ArrayList; import java.util.List; diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryHelper.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryHelper.java index a8972883442..880d68565d4 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryHelper.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryHelper.java @@ -14,10 +14,15 @@ package org.finos.legend.engine.persistence.components.relational.bigquery.executor; -import com.google.cloud.bigquery.*; -import org.finos.legend.engine.persistence.components.logicalplan.datasets.*; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.FieldList; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableId; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Dataset; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.DatasetDefinition; import org.finos.legend.engine.persistence.components.logicalplan.datasets.Field; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.FieldType; +import org.finos.legend.engine.persistence.components.logicalplan.datasets.SchemaDefinition; import org.finos.legend.engine.persistence.components.relational.executor.RelationalExecutionHelper; import org.finos.legend.engine.persistence.components.relational.sql.DataTypeMapping; import org.finos.legend.engine.persistence.components.relational.sql.JdbcPropertiesToLogicalDataTypeMapping; @@ -28,7 +33,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; public class BigQueryHelper implements RelationalExecutionHelper { diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryTransactionManager.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryTransactionManager.java index 6a41ed2bd0e..8368e47a669 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryTransactionManager.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/executor/BigQueryTransactionManager.java @@ -14,8 +14,21 @@ package org.finos.legend.engine.persistence.components.relational.bigquery.executor; -import com.google.cloud.bigquery.*; -import java.util.*; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.ConnectionProperty; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FieldValueList; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.JobInfo; +import com.google.cloud.bigquery.QueryJobConfiguration; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; public class BigQueryTransactionManager { @@ -57,7 +70,7 @@ public void commitTransaction() throws InterruptedException { if (this.sessionId != null) { - executeSql("COMMIT TRANSACTION"); + executeSql("COMMIT TRANSACTION"); } } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchStartTimestampVisitor.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchStartTimestampVisitor.java index d38acab84ea..8bece9ac7b8 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchStartTimestampVisitor.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/visitor/BatchStartTimestampVisitor.java @@ -14,7 +14,10 @@ package org.finos.legend.engine.persistence.components.relational.bigquery.sql.visitor; -import org.finos.legend.engine.persistence.components.logicalplan.values.*; +import org.finos.legend.engine.persistence.components.logicalplan.values.BatchStartTimestamp; +import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionImpl; +import org.finos.legend.engine.persistence.components.logicalplan.values.FunctionName; +import org.finos.legend.engine.persistence.components.logicalplan.values.StringValue; import org.finos.legend.engine.persistence.components.physicalplan.PhysicalPlanNode; import org.finos.legend.engine.persistence.components.relational.ansi.sql.visitors.FunctionVisitor; import org.finos.legend.engine.persistence.components.transformer.LogicalPlanVisitor; diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/CreateTable.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/CreateTable.java index 59ca50999d2..6f687dbe82f 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/CreateTable.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/main/java/org/finos/legend/engine/persistence/components/relational/bigquery/sqldom/schemaops/statements/CreateTable.java @@ -29,7 +29,11 @@ import java.util.ArrayList; import java.util.List; -import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.*; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.CLOSING_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.COMMA; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.EMPTY; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.OPEN_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.WHITE_SPACE; public class CreateTable implements DDLStatement { diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BigQueryEndToEndTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BigQueryEndToEndTest.java index a6da14f5818..b1790b04086 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BigQueryEndToEndTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/e2e/BigQueryEndToEndTest.java @@ -15,7 +15,12 @@ package org.finos.legend.engine.persistence.components.e2e; import com.google.auth.oauth2.GoogleCredentials; -import com.google.cloud.bigquery.*; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.InsertAllRequest; +import com.google.cloud.bigquery.InsertAllResponse; +import com.google.cloud.bigquery.QueryJobConfiguration; +import com.google.cloud.bigquery.TableId; import com.opencsv.CSVReader; import org.finos.legend.engine.persistence.components.common.DatasetFilter; import org.finos.legend.engine.persistence.components.common.Datasets; @@ -38,7 +43,11 @@ import java.io.IOException; import java.time.*; import java.time.format.DateTimeFormatter; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; public class BigQueryEndToEndTest diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeTest.java index e10a712aa80..1e8dbb810ae 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/ingestmode/IngestModeTest.java @@ -22,7 +22,12 @@ import java.time.Clock; import java.time.ZoneOffset; import java.time.ZonedDateTime; -import java.util.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -46,7 +51,7 @@ public class IngestModeTest String batchTimeInField = "batch_time_in"; String batchTimeOutField = "batch_time_out"; - String[] partitionKeys = new String[]{"biz_date"}; + String[] partitionKeys = new String[] {"biz_date"}; HashMap> partitionFilter = new HashMap>() {{ put("biz_date", new HashSet<>(Arrays.asList("2000-01-01 00:00:00", "2000-01-02 00:00:00"))); @@ -70,97 +75,97 @@ public class IngestModeTest Field batchTimeOut = Field.builder().name(batchTimeOutField).type(FieldType.of(DataType.DATETIME, Optional.empty(), Optional.empty())).build(); SchemaDefinition mainTableSchema = SchemaDefinition.builder() - .addFields(id) - .addFields(name) - .addFields(amount) - .addFields(bizDate) - .addFields(digest) - .addFields(batchIdIn) - .addFields(batchIdOut) - .addFields(batchTimeIn) - .addFields(batchTimeOut) - .build(); + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchIdIn) + .addFields(batchIdOut) + .addFields(batchTimeIn) + .addFields(batchTimeOut) + .build(); SchemaDefinition mainTableSchemaWithBatchIdInNotPrimary = SchemaDefinition.builder() - .addFields(id) - .addFields(name) - .addFields(amount) - .addFields(bizDate) - .addFields(digest) - .addFields(batchIdInNonPrimary) - .addFields(batchIdOut) - .addFields(batchTimeIn) - .addFields(batchTimeOut) - .build(); + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchIdInNonPrimary) + .addFields(batchIdOut) + .addFields(batchTimeIn) + .addFields(batchTimeOut) + .build(); SchemaDefinition mainTableBatchIdBasedSchema = SchemaDefinition.builder() - .addFields(id) - .addFields(name) - .addFields(amount) - .addFields(bizDate) - .addFields(digest) - .addFields(batchIdIn) - .addFields(batchIdOut) - .build(); + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchIdIn) + .addFields(batchIdOut) + .build(); SchemaDefinition mainTableBatchIdBasedSchemaWithBatchIdInNotPrimary = SchemaDefinition.builder() - .addFields(id) - .addFields(name) - .addFields(amount) - .addFields(bizDate) - .addFields(digest) - .addFields(batchIdInNonPrimary) - .addFields(batchIdOut) - .build(); + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchIdInNonPrimary) + .addFields(batchIdOut) + .build(); SchemaDefinition mainTableTimeBasedSchema = SchemaDefinition.builder() - .addFields(id) - .addFields(name) - .addFields(amount) - .addFields(bizDate) - .addFields(digest) - .addFields(batchTimeIn) - .addFields(batchTimeOut) - .build(); + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchTimeIn) + .addFields(batchTimeOut) + .build(); SchemaDefinition mainTableTimeBasedSchemaWithBatchTimeInNotPrimary = SchemaDefinition.builder() - .addFields(id) - .addFields(name) - .addFields(amount) - .addFields(bizDate) - .addFields(digest) - .addFields(batchTimeInNonPrimary) - .addFields(batchTimeOut) - .build(); + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .addFields(batchTimeInNonPrimary) + .addFields(batchTimeOut) + .build(); SchemaDefinition baseTableSchemaWithDigest = SchemaDefinition.builder() - .addFields(id) - .addFields(name) - .addFields(amount) - .addFields(bizDate) - .addFields(digest) - .build(); + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(bizDate) + .addFields(digest) + .build(); SchemaDefinition stagingTableSchemaWithLimitedColumns = SchemaDefinition.builder() - .addFields(id) - .addFields(name) - .addFields(amount) - .addFields(digest) - .build(); + .addFields(id) + .addFields(name) + .addFields(amount) + .addFields(digest) + .build(); String expectedMetadataTableCreateQuery = "CREATE REFERENCE TABLE IF NOT EXISTS batch_metadata" + - "(`table_name` VARCHAR(255)," + - "`batch_start_ts_utc` DATETIME," + - "`batch_end_ts_utc` DATETIME," + - "`batch_status` VARCHAR(32)," + - "`table_batch_id` INTEGER)"; + "(`table_name` VARCHAR(255)," + + "`batch_start_ts_utc` DATETIME," + + "`batch_end_ts_utc` DATETIME," + + "`batch_status` VARCHAR(32)," + + "`table_batch_id` INTEGER)"; String expectedMetadataTableCreateQueryWithUpperCase = "CREATE REFERENCE TABLE IF NOT EXISTS BATCH_METADATA" + - "(`TABLE_NAME` VARCHAR(255)," + - "`BATCH_START_TS_UTC` DATETIME," + - "`BATCH_END_TS_UTC` DATETIME," + - "`BATCH_STATUS` VARCHAR(32)," + - "`TABLE_BATCH_ID` INTEGER)"; + "(`TABLE_NAME` VARCHAR(255)," + + "`BATCH_START_TS_UTC` DATETIME," + + "`BATCH_END_TS_UTC` DATETIME," + + "`BATCH_STATUS` VARCHAR(32)," + + "`TABLE_BATCH_ID` INTEGER)"; protected String expectedMetadataTableIngestQuery = "INSERT INTO batch_metadata (`table_name`, `table_batch_id`, `batch_start_ts_utc`, `batch_end_ts_utc`, `batch_status`) (SELECT 'main',(SELECT COALESCE(MAX(batch_metadata.`table_batch_id`),0)+1 FROM batch_metadata as batch_metadata WHERE UPPER(batch_metadata.`table_name`) = 'MAIN'),PARSE_DATETIME('%Y-%m-%d %H:%M:%S','2000-01-01 00:00:00'),CURRENT_TIMESTAMP(),'DONE')"; @@ -169,49 +174,49 @@ public class IngestModeTest String expectedStagingCleanupQuery = "DELETE FROM `mydb`.`staging` as stage"; String expectedMainTableCreateQuery = "CREATE REFERENCE TABLE IF NOT EXISTS `mydb`.`main`" + - "(`id` INTEGER," + - "`name` VARCHAR(256)," + - "`amount` DOUBLE," + - "`biz_date` DATE," + - "`digest` VARCHAR(256)," + - "`batch_id_in` INTEGER," + - "`batch_id_out` INTEGER," + - "`batch_time_in` DATETIME," + - "`batch_time_out` DATETIME," + - "PRIMARY KEY (`id`, `name`, `batch_id_in`, `batch_time_in`))"; + "(`id` INTEGER," + + "`name` VARCHAR(256)," + + "`amount` DOUBLE," + + "`biz_date` DATE," + + "`digest` VARCHAR(256)," + + "`batch_id_in` INTEGER," + + "`batch_id_out` INTEGER," + + "`batch_time_in` DATETIME," + + "`batch_time_out` DATETIME," + + "PRIMARY KEY (`id`, `name`, `batch_id_in`, `batch_time_in`))"; String expectedMainTableCreateQueryWithUpperCase = "CREATE REFERENCE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + - "(`ID` INTEGER," + - "`NAME` VARCHAR(256)," + - "`AMOUNT` DOUBLE," + - "`BIZ_DATE` DATE," + - "`DIGEST` VARCHAR(256)," + - "`BATCH_ID_IN` INTEGER," + - "`BATCH_ID_OUT` INTEGER," + - "`BATCH_TIME_IN` DATETIME," + - "`BATCH_TIME_OUT` DATETIME," + - "PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`, `BATCH_TIME_IN`))"; + "(`ID` INTEGER," + + "`NAME` VARCHAR(256)," + + "`AMOUNT` DOUBLE," + + "`BIZ_DATE` DATE," + + "`DIGEST` VARCHAR(256)," + + "`BATCH_ID_IN` INTEGER," + + "`BATCH_ID_OUT` INTEGER," + + "`BATCH_TIME_IN` DATETIME," + + "`BATCH_TIME_OUT` DATETIME," + + "PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`, `BATCH_TIME_IN`))"; String expectedMainTableBatchIdBasedCreateQuery = "CREATE REFERENCE TABLE IF NOT EXISTS `mydb`.`main`(" + - "`id` INTEGER,`name` VARCHAR(256),`amount` DOUBLE,`biz_date` DATE,`digest` VARCHAR(256)," + - "`batch_id_in` INTEGER,`batch_id_out` INTEGER,PRIMARY KEY (`id`, `name`, `batch_id_in`))"; + "`id` INTEGER,`name` VARCHAR(256),`amount` DOUBLE,`biz_date` DATE,`digest` VARCHAR(256)," + + "`batch_id_in` INTEGER,`batch_id_out` INTEGER,PRIMARY KEY (`id`, `name`, `batch_id_in`))"; String expectedMainTableBatchIdBasedCreateQueryWithUpperCase = "CREATE REFERENCE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + - "(`ID` INTEGER,`NAME` VARCHAR(256),`AMOUNT` DOUBLE,`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," + - "`BATCH_ID_IN` INTEGER,`BATCH_ID_OUT` INTEGER,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`))"; + "(`ID` INTEGER,`NAME` VARCHAR(256),`AMOUNT` DOUBLE,`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," + + "`BATCH_ID_IN` INTEGER,`BATCH_ID_OUT` INTEGER,PRIMARY KEY (`ID`, `NAME`, `BATCH_ID_IN`))"; String expectedMainTableTimeBasedCreateQuery = "CREATE REFERENCE TABLE IF NOT EXISTS `mydb`.`main`(" + - "`id` INTEGER,`name` VARCHAR(256),`amount` DOUBLE,`biz_date` DATE,`digest` VARCHAR(256)," + - "`batch_time_in` DATETIME,`batch_time_out` DATETIME,PRIMARY KEY (`id`, `name`, `batch_time_in`))"; + "`id` INTEGER,`name` VARCHAR(256),`amount` DOUBLE,`biz_date` DATE,`digest` VARCHAR(256)," + + "`batch_time_in` DATETIME,`batch_time_out` DATETIME,PRIMARY KEY (`id`, `name`, `batch_time_in`))"; String expectedMainTableTimeBasedCreateQueryWithUpperCase = "CREATE REFERENCE TABLE IF NOT EXISTS `MYDB`.`MAIN`" + - "(`ID` INTEGER,`NAME` VARCHAR(256),`AMOUNT` DOUBLE,`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," + - "`BATCH_TIME_IN` DATETIME,`BATCH_TIME_OUT` DATETIME,PRIMARY KEY (`ID`, `NAME`, `BATCH_TIME_IN`))"; + "(`ID` INTEGER,`NAME` VARCHAR(256),`AMOUNT` DOUBLE,`BIZ_DATE` DATE,`DIGEST` VARCHAR(256)," + + "`BATCH_TIME_IN` DATETIME,`BATCH_TIME_OUT` DATETIME,PRIMARY KEY (`ID`, `NAME`, `BATCH_TIME_IN`))"; public void assertIfListsAreSameIgnoringOrder(List first, List second) { assertTrue(first.size() == second.size() && - first.stream().sorted().collect(Collectors.toList()) - .equals(second.stream().sorted().collect(Collectors.toList()))); + first.stream().sorted().collect(Collectors.toList()) + .equals(second.stream().sorted().collect(Collectors.toList()))); } } diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/CreateTableTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/CreateTableTest.java index f9da819aa34..d7707d2c712 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/CreateTableTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/logicalplan/operations/CreateTableTest.java @@ -26,7 +26,10 @@ import java.util.List; -import static org.finos.legend.engine.persistence.components.BaseTestUtils.*; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithAllColumns; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithClusteringAndPartitionKey; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithClusteringKey; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.schemaWithPartitionKey; public class CreateTableTest { @@ -35,12 +38,12 @@ public class CreateTableTest public void testCreateTable() { DatasetDefinition dataset = DatasetDefinition.builder() - .database("my_db") - .group("my_schema") - .name("my_table") - .alias("my_alias") - .schema(schemaWithAllColumns) - .build(); + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithAllColumns) + .build(); Operation create = Create.of(true, dataset); LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get()); @@ -90,12 +93,12 @@ public void testCreateTable() public void testCreateTableWithUpperCase() { DatasetDefinition dataset = DatasetDefinition.builder() - .database("my_db") - .group("my_schema") - .name("my_table") - .alias("my_alias") - .schema(schemaWithAllColumns) - .build(); + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithAllColumns) + .build(); Operation create = Create.of(true, dataset); LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get(), TransformOptions.builder().addOptimizers(new UpperCaseOptimizer()).build()); @@ -146,12 +149,12 @@ public void testCreateTableWithUpperCase() public void testCreateTableWithClusteringKey() { DatasetDefinition dataset = DatasetDefinition.builder() - .database("my_db") - .group("my_schema") - .name("my_table") - .alias("my_alias") - .schema(schemaWithClusteringKey) - .build(); + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithClusteringKey) + .build(); Operation create = Create.of(true, dataset); LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get()); @@ -172,12 +175,12 @@ public void testCreateTableWithClusteringKey() public void testCreateTableWithClusteringKeyWithUpperCase() { DatasetDefinition dataset = DatasetDefinition.builder() - .database("my_db") - .group("my_schema") - .name("my_table") - .alias("my_alias") - .schema(schemaWithClusteringKey) - .build(); + .database("my_db") + .group("my_schema") + .name("my_table") + .alias("my_alias") + .schema(schemaWithClusteringKey) + .build(); Operation create = Create.of(true, dataset); LogicalPlan logicalPlan = LogicalPlan.builder().addOps(create).build(); RelationalTransformer transformer = new RelationalTransformer(BigQuerySink.get(), TransformOptions.builder().addOptimizers(new UpperCaseOptimizer()).build()); diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/DataTypeMappingTest.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/DataTypeMappingTest.java index 9525d942b40..ca01554739c 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/DataTypeMappingTest.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-bigquery/src/test/java/org/finos/legend/engine/persistence/components/relational/bigquery/sql/DataTypeMappingTest.java @@ -19,7 +19,41 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import static org.finos.legend.engine.persistence.components.BaseTestUtils.*; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBigint; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBinary; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBool; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBoolean; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBytes; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colBytesWithLength; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colChar; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colCharacter; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDate; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDatetime; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDecimal; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDecimalWithPrecision; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDecimalWithScale; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colDouble; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colFloat; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colFloat64; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colInt; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colInt64; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colInteger; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colJson; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colLongVarBinary; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colLongVarchar; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colLongtext; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colNumber; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colNumeric; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colReal; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colSmallint; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colString; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colStringWithLength; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colText; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colTime; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colTimestamp; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colTinyint; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colVarBinary; +import static org.finos.legend.engine.persistence.components.BaseTestUtils.colVarchar; public class DataTypeMappingTest { diff --git a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/values/WindowFunction.java b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/values/WindowFunction.java index 05f3b4dfe09..243caae8648 100644 --- a/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/values/WindowFunction.java +++ b/legend-engine-xt-persistence-component/legend-engine-xt-persistence-component-relational-core/src/main/java/org/finos/legend/engine/persistence/components/relational/sqldom/schemaops/values/WindowFunction.java @@ -20,8 +20,13 @@ import java.util.ArrayList; import java.util.List; -import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.*; -import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.*; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.ORDER_BY; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.OVER; +import static org.finos.legend.engine.persistence.components.relational.sqldom.common.Clause.PARTITION_BY; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.CLOSING_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.COMMA; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.OPEN_PARENTHESIS; +import static org.finos.legend.engine.persistence.components.relational.sqldom.utils.SqlGenUtils.WHITE_SPACE; public class WindowFunction extends Value {