Implement ConceptColumnSelect conversion (#3465)

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java

@@ -15,6 +15,8 @@
 import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo;
 import com.bakdata.conquery.models.types.ResultType;
 import com.bakdata.conquery.models.types.SemanticType;
+import com.bakdata.conquery.sql.conversion.model.select.ConceptColumnSelectConverter;
+import com.bakdata.conquery.sql.conversion.model.select.SelectConverter;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.dropwizard.validation.ValidationMethod;
 import lombok.Data;
@@ -67,4 +69,8 @@ public ResultType<?> getResultType() {
 		return new ResultType.ListT<>(ResultType.StringT.INSTANCE);
 	}
 
+	@Override
+	public SelectConverter<ConceptColumnSelect> createConverter() {
+		return new ConceptColumnSelectConverter();
+	}
 }

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java

@@ -42,7 +42,7 @@ public ConversionContext convert(CQExternal external, ConversionContext context)
 										 .toList();
 
 		Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal");
-		QueryStep externalStep = QueryStep.createUnionStep(unions, CQ_EXTERNAL_CTE_NAME, Collections.emptyList());
+		QueryStep externalStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_CTE_NAME, Collections.emptyList());
 		return context.withQueryStep(externalStep);
 	}
 

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java

@@ -12,6 +12,7 @@
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
+import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode;
 import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
@@ -153,9 +154,10 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce
 
 		// convert selects
 		SelectContext<Connector, ConnectorSqlTables> selectContext = SelectContext.create(cqTable, ids, tablesValidityDate, connectorTables, conversionContext);
-		List<ConnectorSqlSelects> allSelectsForTable = cqTable.getSelects().stream()
-															  .map(select -> select.createConverter().connectorSelect(select, selectContext))
-															  .toList();
+		List<ConnectorSqlSelects> allSelectsForTable = new ArrayList<>();
+		ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext);
+		allSelectsForTable.add(conceptColumnSelect);
+		cqTable.getSelects().stream().map(select -> select.createConverter().connectorSelect(select, selectContext)).forEach(allSelectsForTable::add);
 
 		return CQTableContext.builder()
 							 .ids(ids)
@@ -275,4 +277,12 @@ private static Optional<SqlFilters> getDateRestriction(ConversionContext context
 		));
 	}
 
+	private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext<Connector, ConnectorSqlTables> selectContext) {
+		return cqConcept.getSelects().stream()
+						.filter(select -> select instanceof ConceptColumnSelect)
+						.findFirst()
+						.map(select -> select.createConverter().connectorSelect(select, selectContext))
+						.orElse(ConnectorSqlSelects.none());
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java

@@ -58,15 +58,18 @@ private Selects collectSelects(CQTableContext tableContext) {
 	}
 
 	/**
-	 * Collects the columns required in {@link ConceptCteStep#AGGREGATION_SELECT}, but also columns additional tables require (like the ones created by the
-	 * {@link SumSqlAggregator}) when distinct-by columns are present. An additional predecessor can contain an N-ary tree of predecessors itself
-	 * (like all {@link QueryStep}s), so we want to look for the deepest preceding QueryStep leafs and collect their {@link ConnectorSqlSelects},
-	 * because they expect this CTE to contain all their {@link SqlSelect#requiredColumns()}.
+	 * Collects the columns required in {@link ConceptCteStep#AGGREGATION_SELECT}, the optional connector column, but also columns additional tables require
+	 * (like the ones created by the {@link SumSqlAggregator}) when distinct-by columns are present. An additional predecessor can contain an N-ary tree of
+	 * predecessors itself (like all {@link QueryStep}s), so we want to look for the deepest preceding QueryStep leafs and collect their
+	 * {@link ConnectorSqlSelects}, because they expect this CTE to contain all their {@link SqlSelect#requiredColumns()}.
 	 */
 	private static List<SqlSelect> collectSelects(ConnectorSqlSelects sqlSelects) {
 		return Stream.concat(
-							 sqlSelects.getAggregationSelects().stream(),
-							 sqlSelects.getAdditionalPredecessor().map(EventFilterCte::collectDeepestPredecessorsColumns).orElse(Stream.empty())
+							 sqlSelects.getConnectorColumn().stream(),
+							 Stream.concat(
+									 sqlSelects.getAggregationSelects().stream(),
+									 sqlSelects.getAdditionalPredecessor().map(EventFilterCte::collectDeepestPredecessorsColumns).orElse(Stream.empty())
+							 )
 					 )
 					 .toList();
 	}

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java

@@ -5,23 +5,21 @@
 import java.time.temporal.ChronoUnit;
 import java.util.List;
 import java.util.Objects;
-import java.util.stream.Collectors;
 
 import com.bakdata.conquery.models.common.CDateSet;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
-import com.bakdata.conquery.models.error.ConqueryError;
 import com.bakdata.conquery.sql.conversion.SharedAliases;
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
 import org.jooq.Condition;
 import org.jooq.DataType;
 import org.jooq.Field;
-import org.jooq.Param;
 import org.jooq.Record;
 import org.jooq.Table;
 import org.jooq.impl.DSL;
+import org.jooq.impl.SQLDataType;
 
 public class HanaSqlFunctionProvider implements SqlFunctionProvider {
 
@@ -43,6 +41,10 @@ public String getMaxDateExpression() {
 
 	@Override
 	public <T> Field<T> cast(Field<?> field, DataType<T> type) {
+		// HANA would require an explicit length param when using CAST with varchar type, TO_VARCHAR does not require this
+		if (type == SQLDataType.VARCHAR) {
+			return DSL.function("TO_VARCHAR", type.getType(), field);
+		}
 		return DSL.function(
 				"CAST",
 				type.getType(),
@@ -154,20 +156,29 @@ public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) {
 	}
 
 	@Override
-	public Field<String> daterangeStringAggregation(ColumnDateRange columnDateRange) {
-
-		Field<String> stringAggregation = DSL.field(
+	public Field<String> stringAggregation(Field<String> stringField, Field<String> delimiter, List<Field<?>> orderByFields) {
+		return DSL.field(
 				"{0}({1}, {2} {3})",
 				String.class,
 				DSL.keyword("STRING_AGG"),
+				stringField,
+				delimiter,
+				DSL.orderBy(orderByFields)
+		);
+	}
+
+	@Override
+	public Field<String> daterangeStringAggregation(ColumnDateRange columnDateRange) {
+
+		Field<String> stringAggregation = stringAggregation(
 				daterangeStringExpression(columnDateRange),
 				DSL.toChar(DELIMITER),
-				DSL.orderBy(columnDateRange.getStart())
+				List.of(columnDateRange.getStart())
 		);
 
 		// encapsulate all ranges (including empty ranges) within curly braces
 		return DSL.when(stringAggregation.isNull(), DSL.val("{}"))
-				  .otherwise(DSL.field("'{' || {0} || '}'", String.class, stringAggregation));
+				  .otherwise(encloseInCurlyBraces(stringAggregation));
 	}
 
 	@Override
@@ -180,9 +191,8 @@ public Field<String> daterangeStringExpression(ColumnDateRange columnDateRange)
 		Field<Date> startDate = columnDateRange.getStart();
 		Field<Date> endDate = columnDateRange.getEnd();
 
-		Param<Integer> dateLength = DSL.val(DEFAULT_DATE_FORMAT.length());
-		Field<String> startDateExpression = toVarcharField(startDate, dateLength);
-		Field<String> endDateExpression = toVarcharField(endDate, dateLength);
+		Field<String> startDateExpression = cast(startDate, SQLDataType.VARCHAR);
+		Field<String> endDateExpression = cast(endDate, SQLDataType.VARCHAR);
 
 		Field<String> withMinDateReplaced = replace(startDateExpression, MIN_DATE_VALUE, MINUS_INFINITY_SIGN);
 		Field<String> withMaxDateReplaced = replace(endDateExpression, MAX_DATE_VALUE, INFINITY_SIGN);
@@ -278,14 +288,6 @@ public Field<String> yearQuarter(Field<Date> dateField) {
 		return DSL.function("QUARTER", String.class, dateField);
 	}
 
-	@Override
-	public Field<Object[]> asArray(List<Field<?>> fields) {
-		String arrayExpression = fields.stream()
-									   .map(Field::toString)
-									   .collect(Collectors.joining(", ", "array(", ")"));
-		return DSL.field(arrayExpression, Object[].class);
-	}
-
 	@Override
 	public Field<Date> addDays(Field<Date> dateColumn, Field<Integer> amountOfDays) {
 		return DSL.function(
@@ -296,17 +298,6 @@ public Field<Date> addDays(Field<Date> dateColumn, Field<Integer> amountOfDays)
 		);
 	}
 
-	private Field<String> toVarcharField(Field<Date> startDate, Param<Integer> dateExpressionLength) {
-		return DSL.field(
-				"{0}({1} {2}({3}))",
-				String.class,
-				DSL.keyword("CAST"),
-				startDate,
-				DSL.keyword("AS VARCHAR"),
-				dateExpressionLength
-		);
-	}
-
 	private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) {
 
 		String startDateExpression = MIN_DATE_VALUE;

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java

@@ -23,7 +23,6 @@
 import org.jooq.Record;
 import org.jooq.Table;
 import org.jooq.impl.DSL;
-import com.google.common.base.Preconditions;
 import org.jooq.impl.SQLDataType;
 
 /**
@@ -165,6 +164,18 @@ public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) {
 						.build();
 	}
 
+	@Override
+	public Field<String> stringAggregation(Field<String> stringField, Field<String> delimiter, List<Field<?>> orderByFields) {
+		return DSL.field(
+				"{0}({1}, {2} {3})",
+				String.class,
+				DSL.keyword("string_agg"),
+				stringField,
+				delimiter,
+				DSL.orderBy(orderByFields)
+		);
+	}
+
 	@Override
 	public Field<String> daterangeStringAggregation(ColumnDateRange columnDateRange) {
 		Field<Object> asMultirange = rangeAgg(columnDateRange);
@@ -246,14 +257,6 @@ public Field<String> yearQuarter(Field<Date> dateField) {
 		);
 	}
 
-	@Override
-	public Field<Object[]> asArray(List<Field<?>> fields) {
-		String arrayExpression = fields.stream()
-									   .map(Field::toString)
-									   .collect(Collectors.joining(", ", "array[", "]"));
-		return DSL.field(arrayExpression, Object[].class);
-	}
-
 	@Override
 	public Field<Date> toDateField(String dateValue) {
 		return DSL.field("{0}::{1}", Date.class, DSL.val(dateValue), DSL.keyword("date"));

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java

@@ -3,6 +3,7 @@
 import java.sql.Date;
 import java.time.temporal.ChronoUnit;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import com.bakdata.conquery.apiv1.query.concept.filter.CQTable;
 import com.bakdata.conquery.models.common.CDateSet;
@@ -11,6 +12,7 @@
 import com.bakdata.conquery.sql.conversion.SharedAliases;
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
+import com.bakdata.conquery.sql.execution.ResultSetProcessor;
 import org.jooq.Condition;
 import org.jooq.DataType;
 import org.jooq.Field;
@@ -27,6 +29,7 @@ public interface SqlFunctionProvider {
 	String DEFAULT_DATE_FORMAT = "yyyy-mm-dd";
 	String INFINITY_SIGN = "∞";
 	String MINUS_INFINITY_SIGN = "-∞";
+	String SQL_UNIT_SEPARATOR = " || '%s' || ".formatted(ResultSetProcessor.UNIT_SEPARATOR);
 
 	String getMinDateExpression();
 
@@ -95,6 +98,8 @@ public interface SqlFunctionProvider {
 	 */
 	QueryStep unnestValidityDate(QueryStep predecessor, String cteName);
 
+	Field<String> stringAggregation(Field<String> stringField, Field<String> delimiter, List<Field<?>> orderByFields);
+
 	/**
 	 * Aggregates the start and end columns of the validity date of entries into one compound string expression.
 	 * <p>
@@ -122,7 +127,7 @@ public interface SqlFunctionProvider {
 
 	Field<Date> addDays(Field<Date> dateColumn, Field<Integer> amountOfDays);
 
-	<T> Field<T> first(Field<T>  field, List<Field<?>> orderByColumn);
+	<T> Field<T> first(Field<T> field, List<Field<?>> orderByColumn);
 
 	<T> Field<T> last(Field<T> column, List<Field<?>> orderByColumns);
 
@@ -135,7 +140,15 @@ public interface SqlFunctionProvider {
 	 */
 	Field<String> yearQuarter(Field<Date> dateField);
 
-	Field<Object[]> asArray(List<Field<?>> fields);
+	default Field<String> concat(List<Field<String>> fields) {
+		String concatenated = fields.stream()
+									// if a field is null, the whole concatenation would be null - but we just want to skip this field in this case,
+									// thus concat an empty string
+									.map(field -> DSL.when(field.isNull(), DSL.val("")).otherwise(field))
+									.map(Field::toString)
+									.collect(Collectors.joining(SQL_UNIT_SEPARATOR));
+		return DSL.field(concatenated, String.class);
+	}
 
 	default <T> Field<T> least(List<Field<T>> fields) {
 		if (fields.isEmpty()) {

backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java

@@ -154,7 +154,7 @@ private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormQue
 		QueryStep featureTable = form.getTimeCountBefore() > 0 ? createCompleteFeatureTable(predecessorSelects, interval, intRange, totalBoundsStep) : null;
 		QueryStep outcomeTable = form.getTimeCountAfter() > 0 ? createCompleteOutcomeTable(predecessorSelects, interval, intRange, totalBoundsStep) : null;
 
-		return QueryStep.createUnionStep(
+		return QueryStep.createUnionAllStep(
 				Stream.concat(Stream.ofNullable(outcomeTable), Stream.ofNullable(featureTable)).toList(),
 				FormCteStep.COMPLETE.getSuffix(),
 				Collections.emptyList()
@@ -186,7 +186,7 @@ private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution reso
 		QueryStep timeBeforeStep = createFeatureTable(totalBoundsStep, interval, seriesIndex, bounds, ids);
 		QueryStep timeAfterStep = createOutcomeTable(totalBoundsStep, interval, seriesIndex, bounds, ids);
 
-		return QueryStep.createUnionStep(
+		return QueryStep.createUnionAllStep(
 				List.of(timeBeforeStep, timeAfterStep),
 				FormCteStep.stratificationCte(resolution).getSuffix(),
 				Collections.emptyList()

backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java

@@ -51,7 +51,7 @@ protected static QueryStep unionResolutionTables(List<QueryStep> unionSteps, Lis
 																					.build())
 														 .toList();
 
-		return QueryStep.createUnionStep(
+		return QueryStep.createUnionAllStep(
 				withQualifiedSelects,
 				FormCteStep.FULL_STRATIFICATION.getSuffix(),
 				Stream.concat(predecessors.stream(), unionSteps.stream()).toList()

backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java

@@ -3,7 +3,6 @@
 import java.util.Collections;
 import java.util.List;
 
-import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect;
 import com.bakdata.conquery.sql.conversion.model.select.SqlSelect;
 import lombok.Builder;
 import lombok.Singular;
@@ -37,17 +36,31 @@ public class QueryStep {
 	 */
 	@Builder.Default
 	List<QueryStep> union = Collections.emptyList();
+	/**
+	 * Determines if this steps union steps should be unioned using a UNION ALL. Default is true.
+	 */
+	@Builder.Default
+	boolean unionAll = true;
 	/**
 	 * All {@link QueryStep}'s that shall be converted before this {@link QueryStep}.
 	 */
 	@Singular
 	List<QueryStep> predecessors;
 
+	public static QueryStep createUnionAllStep(List<QueryStep> unionSteps, String cteName, List<QueryStep> predecessors) {
+		return createUnionStep(unionSteps, cteName, predecessors, true);
+	}
+
 	public static QueryStep createUnionStep(List<QueryStep> unionSteps, String cteName, List<QueryStep> predecessors) {
+		return createUnionStep(unionSteps, cteName, predecessors, false);
+	}
+
+	private static QueryStep createUnionStep(List<QueryStep> unionSteps, String cteName, List<QueryStep> predecessors, boolean unionAll) {
 		return unionSteps.get(0)
 						 .toBuilder()
 						 .cteName(cteName)
 						 .union(unionSteps.subList(1, unionSteps.size()))
+						 .unionAll(unionAll)
 						 .predecessors(predecessors)
 						 .build();
 	}

backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java

@@ -48,7 +48,7 @@ public Select<Record> toSelectQuery(QueryStep queryStep) {
 		if (!queryStep.isUnion()) {
 			return ordered;
 		}
-		return unionAll(queryStep, ordered);
+		return union(queryStep, ordered);
 	}
 
 	private List<CommonTableExpression<Record>> constructPredecessorCteList(QueryStep queryStep) {
@@ -86,15 +86,15 @@ private Select<Record> toSelectStep(QueryStep queryStep) {
 		}
 
 		if (queryStep.isUnion()) {
-			selectStep = unionAll(queryStep, selectStep);
+			selectStep = union(queryStep, selectStep);
 		}
 
 		return selectStep;
 	}
 
-	private Select<Record> unionAll(QueryStep queryStep, Select<Record> base) {
+	private Select<Record> union(QueryStep queryStep, Select<Record> base) {
 		for (QueryStep unionStep : queryStep.getUnion()) {
-			base = base.unionAll(toSelectStep(unionStep));
+			base = queryStep.isUnionAll() ? base.unionAll(toSelectStep(unionStep)) : base.union(toSelectStep(unionStep));
 		}
 		return base;
 	}