diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchRules.java b/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchRules.java index 23eb96bed1b..3f64e23493d 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchRules.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/OpenSearchRules.java @@ -7,10 +7,14 @@ import com.google.common.collect.ImmutableList; import java.util.List; -import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.calcite.plan.RelOptRule; public class OpenSearchRules { - public static final List OPEN_SEARCH_OPT_RULES = ImmutableList.of(); + private static final PPLAggregateConvertRule AGGREGATE_CONVERT_RULE = + PPLAggregateConvertRule.Config.SUM_CONVERTER.toRule(); + + public static final List OPEN_SEARCH_OPT_RULES = + ImmutableList.of(AGGREGATE_CONVERT_RULE); // prevent instantiation private OpenSearchRules() {} diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java b/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java new file mode 100644 index 00000000000..726244663fd --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java @@ -0,0 +1,321 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.sql.calcite.plan; + +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.IntStream; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.runtime.PairList; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.mapping.Mappings; +import org.apache.commons.lang3.tuple.Pair; +import org.immutables.value.Value; + +/** + * Planner rule that converts specific aggCall to a more efficient expressions, which includes: + * + *

- SUM(FIELD + NUMBER) -> SUM(FIELD) + NUMBER * COUNT() + * + *

- SUM(FIELD - NUMBER) -> SUM(FIELD) - NUMBER * COUNT() + * + *

- SUM(FIELD * NUMBER) -> SUM(FIELD) * NUMBER + * + *

- SUM(FIELD / NUMBER) -> SUM(FIELD) / NUMBER, Don't support this because of precision issue + * + *

TODO: + * + *

- AVG/MAX/MIN(FIELD [+|-|*|+|/] NUMBER) -> AVG/MAX/MIN(FIELD) [+|-|*|+|/] NUMBER + */ +@Value.Enclosing +public class PPLAggregateConvertRule extends RelRule { + + /** Creates a OpenSearchAggregateConvertRule. */ + protected PPLAggregateConvertRule(Config config) { + super(config); + } + + @Override + public void onMatch(RelOptRuleCall call) { + if (call.rels.length == 2) { + final LogicalAggregate aggregate = call.rel(0); + final LogicalProject project = call.rel(1); + apply(call, aggregate, project); + } else { + throw new AssertionError( + String.format( + "The length of rels should be %s but got %s", + this.operands.size(), call.rels.length)); + } + } + + public void apply(RelOptRuleCall call, LogicalAggregate aggregate, LogicalProject project) { + + final RelBuilder relBuilder = call.builder(); + final RexBuilder rexBuilder = aggregate.getCluster().getRexBuilder(); + relBuilder.push(project.getInput()); + + /* + Build new projects with fields to be used in the converted agg call. + Need to build this project in advance since building converted agg call has dependency on it. + */ + List aggCalls = aggregate.getAggCallList(); + final List newChildProjects = new ArrayList<>(project.getProjects()); + List convertedAggCallArgs = + aggCalls.stream() + .filter(aggCall -> isConvertableAggCall(aggCall, project)) + .map( + aggCall -> { + RexInputRef rexRef = + getFieldAndLiteral(project.getProjects().get(aggCall.getArgList().getFirst())) + .getLeft(); + // Don't remove elements in the child project since we don't know if it will be + // used by + // other aggCall, will handle unused projects later + int ref = newChildProjects.indexOf(rexRef); + if (ref == -1) { + ref = newChildProjects.size(); + newChildProjects.add(rexRef); + } + return ref; + }) + .toList(); + relBuilder.project(newChildProjects); + RelNode newInput = relBuilder.peek(); + + /* Build converted agg call and its parent projects */ + int convertedAggCallCnt = 0; + final int groupSetOffset = aggregate.getGroupSet().cardinality(); + final List distinctAggregateCalls = new ArrayList<>(); + final PairList newExprOnAggCall = PairList.of(); + for (int i = 0; i < aggregate.getAggCallList().size(); i++) { + AggregateCall aggCall = aggregate.getAggCallList().get(i); + if (isConvertableAggCall(aggCall, project)) { + // The arg ref of convertable aggCall starts at the end of the project + int argRef = convertedAggCallArgs.get(convertedAggCallCnt++); + AggregateCall sumCall = + AggregateCall.create( + aggCall.getParserPosition(), + aggCall.getAggregation(), + aggCall.isDistinct(), + aggCall.isApproximate(), + aggCall.ignoreNulls(), + aggCall.rexList, + ImmutableList.of(argRef), + aggCall.filterArg, + aggCall.distinctKeys, + aggCall.collation, + aggregate.getGroupCount(), + newInput, // Note: must be the new Project + null, // The type will be inferred. + aggCall.getName() + "_SUM"); + int sumCallRef = putToDistinctAggregateCalls(distinctAggregateCalls, sumCall); + + final Function> literalConverterProvider; + RexCall rexCall = (RexCall) project.getProjects().get(aggCall.getArgList().getFirst()); + if (rexCall.getOperator().kind == SqlKind.PLUS + || rexCall.getOperator().kind == SqlKind.MINUS) { + AggregateCall countCall = + AggregateCall.create( + aggCall.getParserPosition(), + SqlStdOperatorTable.COUNT, + aggCall.isDistinct(), + aggCall.isApproximate(), + aggCall.ignoreNulls(), + aggCall.rexList, + ImmutableList.of(argRef), + aggCall.filterArg, + aggCall.distinctKeys, + aggCall.collation, + aggregate.getGroupCount(), + newInput, + null, // The type will be inferred. + aggCall.getName() + "_COUNT"); + int countCallRef = putToDistinctAggregateCalls(distinctAggregateCalls, countCall); + literalConverterProvider = + input -> + literal -> + rexBuilder.makeCall( + aggCall.getType(), + SqlStdOperatorTable.MULTIPLY, + List.of( + rexBuilder.makeInputRef(input, groupSetOffset + countCallRef), + literal)); + } else { + literalConverterProvider = input -> literal -> literal; + } + newExprOnAggCall.add( + input -> { + Function fieldConverter = + field -> rexBuilder.makeInputRef(input, groupSetOffset + sumCallRef); + Function literalConverter = literalConverterProvider.apply(input); + List operands = + List.of( + convertToNewOperand( + rexCall.getOperands().getFirst(), fieldConverter, literalConverter), + convertToNewOperand( + rexCall.getOperands().getLast(), fieldConverter, literalConverter)); + return rexBuilder.makeCall(aggCall.getType(), rexCall.getOperator(), operands); + }, + aggCall.getName()); + } else { + int callRef = putToDistinctAggregateCalls(distinctAggregateCalls, aggCall); + newExprOnAggCall.add( + input -> rexBuilder.makeInputRef(input, groupSetOffset + callRef), aggCall.getName()); + } + } + + /* Eliminate unused fields in the child project */ + ImmutableBitSet newGroupSet = aggregate.getGroupSet(); + ; + ImmutableList newGroupSets = aggregate.getGroupSets(); + ; + final Set fieldsUsed = + RelOptUtil.getAllFields2(aggregate.getGroupSet(), distinctAggregateCalls); + if (fieldsUsed.size() < newChildProjects.size()) { + // Some fields are computed but not used. Prune them. + final Map sourceFieldToTargetFieldMap = new HashMap<>(); + for (int source : fieldsUsed) { + sourceFieldToTargetFieldMap.put(source, sourceFieldToTargetFieldMap.size()); + } + newGroupSet = aggregate.getGroupSet().permute(sourceFieldToTargetFieldMap); + newGroupSets = + ImmutableBitSet.ORDERING.immutableSortedCopy( + ImmutableBitSet.permute(aggregate.getGroupSets(), sourceFieldToTargetFieldMap)); + final Mappings.TargetMapping targetMapping = + Mappings.target(sourceFieldToTargetFieldMap, newChildProjects.size(), fieldsUsed.size()); + final List oldAggregateCalls = new ArrayList<>(distinctAggregateCalls); + distinctAggregateCalls.clear(); + for (AggregateCall aggregateCall : oldAggregateCalls) { + distinctAggregateCalls.add(aggregateCall.transform(targetMapping)); + } + // Project the used fields + relBuilder.project(relBuilder.fields(fieldsUsed.stream().toList())); + } + + /* Build the final project-aggregate-project after eliminating unused fields */ + relBuilder.aggregate(relBuilder.groupKey(newGroupSet, newGroupSets), distinctAggregateCalls); + List parentProjects = + new ArrayList<>(relBuilder.fields(IntStream.range(0, groupSetOffset).boxed().toList())); + parentProjects.addAll( + newExprOnAggCall.transform( + (constructor, name) -> + aliasMaybe(relBuilder, constructor.apply(relBuilder.peek()), name))); + relBuilder.project(parentProjects); + call.transformTo(relBuilder.build()); + } + + interface OperatorConstructor { + RexNode apply(RelNode input); + } + + private int putToDistinctAggregateCalls( + List distinctAggregateCalls, AggregateCall aggCall) { + int i = distinctAggregateCalls.indexOf(aggCall); + if (i < 0) { + i = distinctAggregateCalls.size(); + distinctAggregateCalls.add(aggCall); + } + return i; + } + + private boolean isConvertableAggCall(AggregateCall aggCall, Project project) { + return aggCall.getAggregation().getKind() == SqlKind.SUM + && Config.isCallWithLiteral(project.getProjects().get(aggCall.getArgList().getFirst())); + } + + private static Pair getFieldAndLiteral(RexNode node) { + RexCall call = (RexCall) node; + RexNode arg1 = call.getOperands().getFirst(); + RexNode arg2 = call.getOperands().getLast(); + return arg1.getKind() == SqlKind.INPUT_REF + ? Pair.of((RexInputRef) arg1, (RexLiteral) arg2) + : Pair.of((RexInputRef) arg2, (RexLiteral) arg1); + } + + private static RexNode convertToNewOperand( + RexNode operand, + Function fieldConverter, + Function literalConverter) { + if (operand.getKind() == SqlKind.INPUT_REF) { + return fieldConverter.apply(operand); + } else { + return literalConverter.apply(operand); + } + } + + private RexNode aliasMaybe(RelBuilder builder, RexNode node, String alias) { + return alias == null ? node : builder.alias(node, alias); + } + + /** Rule configuration. */ + @Value.Immutable + public interface Config extends RelRule.Config { + Config SUM_CONVERTER = + ImmutablePPLAggregateConvertRule.Config.builder() + .build() + .withOperandSupplier( + b0 -> + b0.operand(LogicalAggregate.class) + .predicate(Config::containsSumAggCall) + .oneInput( + b1 -> + b1.operand(LogicalProject.class) + .predicate(Config::containsCallWithNumber) + .anyInputs())); + + static boolean containsSumAggCall(LogicalAggregate aggregate) { + return aggregate.getAggCallList().stream() + .anyMatch(aggCall -> aggCall.getAggregation().getKind() == SqlKind.SUM); + } + + static boolean containsCallWithNumber(LogicalProject project) { + return project.getProjects().stream().anyMatch(Config::isCallWithLiteral); + } + + private static boolean isCallWithLiteral(RexNode node) { + if (CONVERTABLE_FUNCTIONS.contains(node.getKind()) && node instanceof RexCall call) { + RexNode arg1 = call.getOperands().getFirst(); + RexNode arg2 = call.getOperands().getLast(); + return (arg1.getKind() == SqlKind.INPUT_REF && arg2.getKind() == SqlKind.LITERAL) + || (arg1.getKind() == SqlKind.LITERAL && arg2.getKind() == SqlKind.INPUT_REF); + } + return false; + } + + List CONVERTABLE_FUNCTIONS = + List.of( + SqlKind.PLUS, SqlKind.MINUS, SqlKind.TIMES + // Don't support division because of the issue of integer division + // e.g. (2000 / 3) * 3 = 1998 while 2000 * 3 / 3 = 2000 + // SqlKind.DIVIDE + ); + + @Override + default PPLAggregateConvertRule toRule() { + return new PPLAggregateConvertRule(this); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java index 14c8d8f369e..f2eb2e034ca 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/CalciteToolsHelper.java @@ -88,6 +88,7 @@ import org.apache.calcite.util.Holder; import org.apache.calcite.util.Util; import org.opensearch.sql.calcite.CalcitePlanContext; +import org.opensearch.sql.calcite.plan.OpenSearchRules; import org.opensearch.sql.calcite.plan.Scannable; import org.opensearch.sql.calcite.udf.udaf.NullableSqlAvgAggFunction; @@ -231,10 +232,15 @@ public R perform( final RelOptPlanner planner = createPlanner( prepareContext, Contexts.of(prepareContext.config()), config.getCostFactory()); + registerCustomizedRules(planner); final RelOptCluster cluster = createCluster(planner, rexBuilder); return action.apply(cluster, catalogReader, prepareContext.getRootSchema().plus(), statement); } + private void registerCustomizedRules(RelOptPlanner planner) { + OpenSearchRules.OPEN_SEARCH_OPT_RULES.forEach(planner::addRule); + } + /** * Customize CalcitePreparingStmt. Override {@link CalcitePrepareImpl#getPreparingStmt} and * return {@link OpenSearchCalcitePreparingStmt} diff --git a/core/src/test/java/org/opensearch/sql/calcite/PPLAggregateConvertRuleTest.java b/core/src/test/java/org/opensearch/sql/calcite/PPLAggregateConvertRuleTest.java new file mode 100644 index 00000000000..652ee108aca --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/calcite/PPLAggregateConvertRuleTest.java @@ -0,0 +1,124 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.when; +import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.plan.volcano.VolcanoRuleCall; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalAggregate; +import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.tools.RelBuilder; +import org.apache.calcite.tools.RelBuilder.AggCall; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.sql.calcite.plan.PPLAggregateConvertRule; +import org.opensearch.sql.calcite.utils.CalciteToolsHelper.OpenSearchRelBuilder; + +@ExtendWith(MockitoExtension.class) +public class PPLAggregateConvertRuleTest { + public static final PPLAggregateConvertRule AGGREGATE_CONVERT_RULE = + PPLAggregateConvertRule.Config.SUM_CONVERTER.toRule(); + @Mock VolcanoRuleCall mockedCall; + @Mock RelNode input; + @Mock RelOptCluster cluster; + @Mock RelOptPlanner planner; + RelDataType type = TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT); + RelDataType rowType = TYPE_FACTORY.createStructType(List.of(type, type), List.of("a", "b")); + RexBuilder rexBuilder = new RexBuilder(TYPE_FACTORY); + RelBuilder relBuilder; + + @BeforeEach + public void setUp() throws IllegalAccessException, NoSuchFieldException { + when(cluster.getTypeFactory()).thenReturn(TYPE_FACTORY); + when(cluster.getRexBuilder()).thenReturn(rexBuilder); + when(cluster.traitSet()).thenReturn(RelTraitSet.createEmpty()); + when(cluster.traitSetOf(Convention.NONE)) + .thenReturn(RelTraitSet.createEmpty().replace(Convention.NONE)); + when(cluster.getPlanner()).thenReturn(planner); + when(planner.getExecutor()).thenReturn(null); + + when(input.getCluster()).thenReturn(cluster); + when(input.getRowType()).thenReturn(rowType); + relBuilder = new OpenSearchRelBuilder(null, cluster, null); + when(mockedCall.builder()).thenReturn(relBuilder); + } + + @Test + public void testRuleMatch() { + relBuilder.push(input); + RexNode arg = + rexBuilder.makeCall( + SqlStdOperatorTable.PLUS, + List.of( + new RexInputRef(0, TYPE_FACTORY.createSqlType(SqlTypeName.BIGINT)), + rexBuilder.makeLiteral(10, type))); + AggCall aggCall = relBuilder.aggregateCall(SqlStdOperatorTable.SUM, arg); + LogicalAggregate aggregate = + (LogicalAggregate) + relBuilder.aggregate(relBuilder.groupKey(1), ImmutableList.of(aggCall)).build(); + assert (aggregate.getInput() instanceof LogicalProject); + LogicalProject project = (LogicalProject) aggregate.getInput(); + + // Check the predicate in Config + assertTrue(PPLAggregateConvertRule.Config.containsSumAggCall(aggregate)); + assertTrue(PPLAggregateConvertRule.Config.containsCallWithNumber(project)); + + doAnswer( + invocation -> { + // Check the final plan + RelNode rel = invocation.getArgument(0); + assertTrue( + RelOptUtil.areRowTypesEqual(rel.getRowType(), aggregate.getRowType(), true)); + + assertInstanceOf(LogicalProject.class, rel); + LogicalProject parentProject = (LogicalProject) rel; + assertTrue( + parentProject.getProjects().getLast() instanceof RexCall call + && call.getOperator() == SqlStdOperatorTable.PLUS); + + assertInstanceOf(LogicalAggregate.class, parentProject.getInput()); + LogicalAggregate newAggregate = (LogicalAggregate) parentProject.getInput(); + assertTrue( + newAggregate.getAggCallList().getFirst().getAggregation() + == SqlStdOperatorTable.SUM + && newAggregate.getAggCallList().getLast().getAggregation() + == SqlStdOperatorTable.COUNT); + + assertInstanceOf(LogicalProject.class, newAggregate.getInput()); + LogicalProject childProject = (LogicalProject) newAggregate.getInput(); + assertTrue( + childProject.getProjects().stream().allMatch(rex -> rex instanceof RexInputRef)); + + return null; + }) + .when(mockedCall) + .transformTo(any()); + AGGREGATE_CONVERT_RULE.apply(mockedCall, aggregate, project); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java index af7d748f5cd..e1982b52742 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java @@ -49,8 +49,10 @@ public static void reset() throws IOException { } /** - * Ignore queries that are not supported by Calcite. Ignore q30 because of too much script push - * down, which will cause ResourceMonitor restriction. + * Ignore queries that are not supported by Calcite. + * + *

q30 is ignored because it will trigger ResourceMonitory health check. TODO: should be + * addressed by: https://github.com/opensearch-project/sql/issues/3981 */ protected Set ignored() { return Set.of(29, 30); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 51e1a7ad4bb..8ce6e6d8921 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.calcite.remote; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.util.MatcherUtils.assertJsonEqualsIgnoreId; import java.io.IOException; @@ -171,6 +172,19 @@ public void supportPushDownScriptOnTextField() throws IOException { assertJsonEqualsIgnoreId(expected, result); } + // Only for Calcite, as v2 gets unstable serialized string for function + @Test + public void testExplainOnAggregationWithSumEnhancement() throws IOException { + String expected = loadExpectedPlan("explain_agg_with_sum_enhancement.json"); + assertJsonEqualsIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats sum(balance), sum(balance + 100), sum(balance - 100)," + + " sum(balance * 100), sum(balance / 100) by gender", + TEST_INDEX_BANK))); + } + /** * Executes the PPL query and returns the result as a string with windows-style line breaks * replaced with Unix-style ones. diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index d1ee937e14d..a66def191c1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -44,6 +44,28 @@ public void testStatsSum() throws IOException { verifyDataRows(response, rows(25714837)); } + @Test + public void testStatsSumWithEnhancement() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | stats sum(balance), sum(balance + 100), sum(balance - 100)," + + " sum(balance * 100), sum(balance / 100) by gender", + TEST_INDEX_ACCOUNT)); + verifySchema( + response, + schema("sum(balance)", null, "bigint"), + schema("sum(balance + 100)", null, "bigint"), + schema("sum(balance - 100)", null, "bigint"), + schema("sum(balance * 100)", null, "bigint"), + schema("sum(balance / 100)", null, "bigint"), + schema("gender", null, "string")); + verifyDataRows( + response, + rows(12632310, 12681610, 12583010, 1263231000, 126080, "F"), + rows(13082527, 13133227, 13031827, 1308252700, 130570, "M")); + } + @Test public void testStatsCount() throws IOException { JSONObject response = diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json new file mode 100644 index 00000000000..45d5b4a0544 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0])\n LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)])\n LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum(balance)\":{\"sum\":{\"field\":\"balance\"}},\"sum(balance + 100)_COUNT\":{\"value_count\":{\"field\":\"balance\"}},\"sum(balance / 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHjnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJiaXJ0aGRhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJjaXR5IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAibGFzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1wbG95ZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJtYWxlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBz3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogNywKICAgICAgIm5hbWUiOiAiJDciCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEwMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn10AApmaWVsZFR5cGVzc3IAF2phdmEudXRpbC5MaW5rZWRIYXNoTWFwNMBOXBBswPsCAAFaAAthY2Nlc3NPcmRlcnhyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAANdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lfnEAfgALdAAGU1RSSU5HdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAUeHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+AB94cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ACEAAAAAc3EAfgAAAAAAA3cEAAAAAHh0AAliaXJ0aGRhdGVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cQB+ABV+cQB+AAt0AAlUSU1FU1RBTVB+cQB+ABt0AAREYXRlcQB+ACBzcQB+AAAAAAABdwQAAAAAeHQABmdlbmRlcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAVcQB+ABB+cQB+ABt0AAdLZXl3b3JkcQB+ACB4dAAEY2l0eXEAfgAQdAAIbGFzdG5hbWVxAH4AEHQAB2JhbGFuY2VxAH4ADXQACGVtcGxveWVyc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AAVzdGF0ZXNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJxAH4AMXEAfgAyeHQAA2FnZX5xAH4AC3QAB0lOVEVHRVJ0AAVlbWFpbHNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAEbWFsZX5xAH4AC3QAB0JPT0xFQU54AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.json new file mode 100644 index 00000000000..90ae3beb4d2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_with_sum_enhancement.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0])\n LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)])\n LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0])\n EnumerableAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)_COUNT=[COUNT($1)], sum(balance / 100)=[SUM($2)])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[100], expr#20=[DIVIDE($t7, $t19)], gender=[$t4], balance=[$t7], $f5=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_sum_enhancement.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_sum_enhancement.json new file mode 100644 index 00000000000..e965b5b5dc2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_agg_with_sum_enhancement.json @@ -0,0 +1,17 @@ +{ + "root": { + "name": "ProjectOperator", + "description": { + "fields": "[sum(balance), sum(balance + 100), sum(balance - 100), sum(balance * 100), sum(balance / 100), gender]" + }, + "children": [ + { + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum(balance)\":{\"sum\":{\"field\":\"balance\"}},\"sum(balance + 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAdiYWxhbmNlc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05Hc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAGR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAStxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAGWxhbWJkYSRhZGRCYXNlJDdkOTBhYzAyJDR0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGiQdxM68QzgeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUExxAH4AFQ==\\\"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(balance - 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAdiYWxhbmNlc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05Hc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAGR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAS1xAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAHmxhbWJkYSRzdWJ0cmFjdEJhc2UkN2Q5MGFjMDIkNHQAfShMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADV2cgAyb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25EU0wAAAAAAAAAAAAAAHhwcQB+ADBxAH4AMXEAfgAydAAyb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25EU0x0ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4ANXEAfgA3dAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4AMXQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+ADh0ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AI3NyADlvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvblByb3BlcnRpZXPPPFljm6j5swIAA0wADWN1cnJlbnRab25lSWR0ABJMamF2YS90aW1lL1pvbmVJZDtMAApub3dJbnN0YW50dAATTGphdmEvdGltZS9JbnN0YW50O0wACXF1ZXJ5VHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhlY3V0b3IvUXVlcnlUeXBlO3hwc3IADWphdmEudGltZS5TZXKVXYS6GyJIsgwAAHhwdwIIAHhzcQB+AEV3DQIAAAAAaJB3EzrxDOB4fnIAJW9yZy5vcGVuc2VhcmNoLnNxbC5leGVjdXRvci5RdWVyeVR5cGUAAAAAAAAAABIAAHhxAH4AFHQAA1BQTHEAfgAV\\\"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(balance * 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAdiYWxhbmNlc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05Hc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAGR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAASpxAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAHmxhbWJkYSRtdWx0aXBseUJhc2UkN2Q5MGFjMDIkNHQAfShMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADV2cgAyb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25EU0wAAAAAAAAAAAAAAHhwcQB+ADBxAH4AMXEAfgAydAAyb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25EU0x0ACVsYW1iZGEkbnVsbE1pc3NpbmdIYW5kbGluZyRhNTAwNTI4MSQxdAC8KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4ANXEAfgA3dAA+b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlVHJpRnVuY3Rpb25xAH4AMXQASihMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDtMamF2YS9sYW5nL09iamVjdDspTGphdmEvbGFuZy9PYmplY3Q7cQB+ADh0ABZsYW1iZGEkaW1wbCRhMGZiMzRkNCQxdAD3KExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVCaUZ1bmN0aW9uO0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9GdW5jdGlvblByb3BlcnRpZXM7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3QAuChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtxAH4AI3NyADlvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvblByb3BlcnRpZXPPPFljm6j5swIAA0wADWN1cnJlbnRab25lSWR0ABJMamF2YS90aW1lL1pvbmVJZDtMAApub3dJbnN0YW50dAATTGphdmEvdGltZS9JbnN0YW50O0wACXF1ZXJ5VHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZXhlY3V0b3IvUXVlcnlUeXBlO3hwc3IADWphdmEudGltZS5TZXKVXYS6GyJIsgwAAHhwdwIIAHhzcQB+AEV3DQIAAAAAaJB3EzrxDOB4fnIAJW9yZy5vcGVuc2VhcmNoLnNxbC5leGVjdXRvci5RdWVyeVR5cGUAAAAAAAAAABIAAHhxAH4AFHQAA1BQTHEAfgAV\\\"}\",\"lang\":\"opensearch_compounded_script\"}}},\"sum(balance / 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"v2\\\",\\\"script\\\":\\\"rO0ABXNyADRvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5GdW5jdGlvbkRTTCQzHWCy3iOeynUCAAVMAA12YWwkYXJndW1lbnRzdAAQTGphdmEvdXRpbC9MaXN0O0wADHZhbCRmdW5jdGlvbnQAQExvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9mdW5jdGlvbi9TZXJpYWxpemFibGVUcmlGdW5jdGlvbjtMABB2YWwkZnVuY3Rpb25OYW1ldAA1TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uTmFtZTtMABZ2YWwkZnVuY3Rpb25Qcm9wZXJ0aWVzdAA7TG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMAA52YWwkcmV0dXJuVHlwZXQAJ0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJUeXBlO3hyADBvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5GdW5jdGlvbkV4cHJlc3Npb26yKjDT3HVqewIAAkwACWFyZ3VtZW50c3EAfgABTAAMZnVuY3Rpb25OYW1lcQB+AAN4cHNyABNqYXZhLnV0aWwuQXJyYXlMaXN0eIHSHZnHYZ0DAAFJAARzaXpleHAAAAACdwQAAAACc3IAMW9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLlJlZmVyZW5jZUV4cHJlc3Npb26rRO9cEgeF1gIABEwABGF0dHJ0ABJMamF2YS9sYW5nL1N0cmluZztMAAVwYXRoc3EAfgABTAAHcmF3UGF0aHEAfgALTAAEdHlwZXEAfgAFeHB0AAdiYWxhbmNlc3IAGmphdmEudXRpbC5BcnJheXMkQXJyYXlMaXN02aQ8vs2IBtICAAFbAAFhdAATW0xqYXZhL2xhbmcvT2JqZWN0O3hwdXIAE1tMamF2YS5sYW5nLlN0cmluZzut0lbn6R17RwIAAHhwAAAAAXEAfgANcQB+AA1+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05Hc3IAL29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLkxpdGVyYWxFeHByZXNzaW9uRUIt8IzHgiQCAAFMAAlleHByVmFsdWV0AClMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3hwc3IALm9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLm1vZGVsLkV4cHJJbnRlZ2VyVmFsdWWmbAeZibduQwIAAHhyADVvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS5tb2RlbC5BYnN0cmFjdEV4cHJOdW1iZXJWYWx1ZTT0YurFpzI7AgABTAAFdmFsdWV0ABJMamF2YS9sYW5nL051bWJlcjt4cgAvb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEubW9kZWwuQWJzdHJhY3RFeHByVmFsdWXJa7V2BhREigIAAHhwc3IAEWphdmEubGFuZy5JbnRlZ2VyEuKgpPeBhzgCAAFJAAV2YWx1ZXhyABBqYXZhLmxhbmcuTnVtYmVyhqyVHQuU4IsCAAB4cAAAAGR4c3IAM29yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uTmFtZQuoOE3O9meXAgABTAAMZnVuY3Rpb25OYW1lcQB+AAt4cHQAAS9xAH4ACXNyACFqYXZhLmxhbmcuaW52b2tlLlNlcmlhbGl6ZWRMYW1iZGFvYdCULCk2hQIACkkADmltcGxNZXRob2RLaW5kWwAMY2FwdHVyZWRBcmdzcQB+AA9MAA5jYXB0dXJpbmdDbGFzc3QAEUxqYXZhL2xhbmcvQ2xhc3M7TAAYZnVuY3Rpb25hbEludGVyZmFjZUNsYXNzcQB+AAtMAB1mdW5jdGlvbmFsSW50ZXJmYWNlTWV0aG9kTmFtZXEAfgALTAAiZnVuY3Rpb25hbEludGVyZmFjZU1ldGhvZFNpZ25hdHVyZXEAfgALTAAJaW1wbENsYXNzcQB+AAtMAA5pbXBsTWV0aG9kTmFtZXEAfgALTAATaW1wbE1ldGhvZFNpZ25hdHVyZXEAfgALTAAWaW5zdGFudGlhdGVkTWV0aG9kVHlwZXEAfgALeHAAAAAGdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAXNxAH4AJQAAAAZ1cQB+ACgAAAABc3EAfgAlAAAABnVxAH4AKAAAAAB2cgBEb3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24ub3BlcmF0b3IuYXJ0aG1ldGljLkFyaXRobWV0aWNGdW5jdGlvbnMAAAAAAAAAAAAAAHhwdAA9b3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbnQABWFwcGx5dAA4KExqYXZhL2xhbmcvT2JqZWN0O0xqYXZhL2xhbmcvT2JqZWN0OylMamF2YS9sYW5nL09iamVjdDt0AERvcmcvb3BlbnNlYXJjaC9zcWwvZXhwcmVzc2lvbi9vcGVyYXRvci9hcnRobWV0aWMvQXJpdGhtZXRpY0Z1bmN0aW9uc3QAHGxhbWJkYSRkaXZpZGVCYXNlJDdkOTBhYzAyJDR0AH0oTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlOylMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO3EAfgA1dnIAMm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLkZ1bmN0aW9uRFNMAAAAAAAAAAAAAAB4cHEAfgAwcQB+ADFxAH4AMnQAMm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uRFNMdAAlbGFtYmRhJG51bGxNaXNzaW5nSGFuZGxpbmckYTUwMDUyODEkMXQAvChMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ADVxAH4AN3QAPm9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL1NlcmlhbGl6YWJsZVRyaUZ1bmN0aW9ucQB+ADF0AEooTGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7TGphdmEvbGFuZy9PYmplY3Q7KUxqYXZhL2xhbmcvT2JqZWN0O3EAfgA4dAAWbGFtYmRhJGltcGwkYTBmYjM0ZDQkMXQA9yhMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vU2VyaWFsaXphYmxlQmlGdW5jdGlvbjtMb3JnL29wZW5zZWFyY2gvc3FsL2V4cHJlc3Npb24vZnVuY3Rpb24vRnVuY3Rpb25Qcm9wZXJ0aWVzO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7TG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTspTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL21vZGVsL0V4cHJWYWx1ZTt0ALgoTG9yZy9vcGVuc2VhcmNoL3NxbC9leHByZXNzaW9uL2Z1bmN0aW9uL0Z1bmN0aW9uUHJvcGVydGllcztMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvbW9kZWwvRXhwclZhbHVlO0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7KUxvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS9tb2RlbC9FeHByVmFsdWU7cQB+ACNzcgA5b3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uRnVuY3Rpb25Qcm9wZXJ0aWVzzzxZY5uo+bMCAANMAA1jdXJyZW50Wm9uZUlkdAASTGphdmEvdGltZS9ab25lSWQ7TAAKbm93SW5zdGFudHQAE0xqYXZhL3RpbWUvSW5zdGFudDtMAAlxdWVyeVR5cGV0ACdMb3JnL29wZW5zZWFyY2gvc3FsL2V4ZWN1dG9yL1F1ZXJ5VHlwZTt4cHNyAA1qYXZhLnRpbWUuU2VylV2EuhsiSLIMAAB4cHcCCAB4c3EAfgBFdw0CAAAAAGiQdxM68QzgeH5yACVvcmcub3BlbnNlYXJjaC5zcWwuZXhlY3V0b3IuUXVlcnlUeXBlAAAAAAAAAAASAAB4cQB+ABR0AANQUExxAH4AFQ==\\\"}\",\"lang\":\"opensearch_compounded_script\"}}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + }, + "children": [] + } + ] + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 9f30b3b497c..576f7523dd3 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -28,6 +28,8 @@ import org.apache.calcite.rel.RelFieldCollation.NullDirection; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.core.Aggregate; +import org.apache.calcite.rel.core.AggregateCall; import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalAggregate; @@ -45,6 +47,7 @@ import org.opensearch.search.aggregations.bucket.composite.CompositeAggregationBuilder; import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; +import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.search.sort.ScoreSortBuilder; import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; @@ -120,7 +123,8 @@ public double estimateRowCount(RelMetadataQuery mq) { osIndex.getMaxResultWindow().doubleValue(), (rowCount, action) -> switch (action.type) { - case AGGREGATION -> mq.getRowCount((RelNode) action.digest); + case AGGREGATION -> mq.getRowCount((RelNode) action.digest) + * getAggMultiplier(action); case PROJECT, SORT -> rowCount; // Refer the org.apache.calcite.rel.core.Aggregate.estimateRowCount case COLLAPSE -> rowCount * (1.0 - Math.pow(.5, 1)); @@ -135,6 +139,28 @@ public double estimateRowCount(RelMetadataQuery mq) { (a, b) -> null); } + /** See source in {@link org.apache.calcite.rel.core.Aggregate::computeSelfCost} */ + private static float getAggMultiplier(PushDownAction action) { + // START CALCITE + List aggCalls = ((Aggregate) action.digest).getAggCallList(); + float multiplier = 1f + (float) aggCalls.size() * 0.125f; + for (AggregateCall aggCall : aggCalls) { + if (aggCall.getAggregation().getName().equals("SUM")) { + // Pretend that SUM costs a little bit more than $SUM0, + // to make things deterministic. + multiplier += 0.0125f; + } + } + // END CALCITE + + // For script aggregation, we need to multiply the multiplier by 2.2 to make up the cost. As we + // prefer to have non-script agg push down after optimized by {@link PPLAggregateConvertRule} + if (((AggPushDownAction) action.action).isScriptPushed) { + multiplier *= 2.2f; + } + return multiplier; + } + // TODO: should we consider equivalent among PushDownContexts with different push down sequence? public static class PushDownContext extends ArrayDeque { @@ -346,12 +372,20 @@ public static class AggPushDownAction implements AbstractAction { private Pair, OpenSearchAggregationResponseParser> aggregationBuilder; private final Map extendedTypeMapping; + @Getter private final boolean isScriptPushed; public AggPushDownAction( Pair, OpenSearchAggregationResponseParser> aggregationBuilder, Map extendedTypeMapping) { this.aggregationBuilder = aggregationBuilder; this.extendedTypeMapping = extendedTypeMapping; + this.isScriptPushed = + aggregationBuilder.getLeft().stream().anyMatch(this::isScriptAggBuilder); + } + + private boolean isScriptAggBuilder(AggregationBuilder aggBuilder) { + return aggBuilder instanceof ValuesSourceAggregationBuilder valueSourceAgg + && valueSourceAgg.script() != null; } @Override