Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Transform pipeline aggr test #1027

Merged
merged 6 commits into from
Nov 1, 2023
Merged

Transform pipeline aggr test #1027

Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
af10e33
tester code: pipeline aggr. transform job
n-dohrmann Oct 31, 2023
937f89c
made test case for pipeline aggregator in transform job
n-dohrmann Oct 31, 2023
1b79801
Merge branch 'main' into transform_pipeline_aggr_test
n-dohrmann Oct 31, 2023
c75ca60
removed unnec. test lines
n-dohrmann Nov 1, 2023
1f6620a
Merge branch 'transform_pipeline_aggr_test' of github.com:n-dohrmann/…
n-dohrmann Nov 1, 2023
3445ec9
re-added method call on Transform obj
n-dohrmann Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRunnerIT.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,11 @@ import org.opensearch.script.ScriptType
import org.opensearch.search.aggregations.AggregationBuilders
import org.opensearch.search.aggregations.AggregatorFactories
import org.opensearch.search.aggregations.metrics.ScriptedMetricAggregationBuilder
import org.opensearch.search.aggregations.pipeline.BucketScriptPipelineAggregationBuilder
import java.lang.Integer.min
import java.time.Instant
import java.time.temporal.ChronoUnit
import kotlin.test.assertFailsWith

class TransformRunnerIT : TransformRestTestCase() {

Expand Down Expand Up @@ -688,6 +690,42 @@ class TransformRunnerIT : TransformRestTestCase() {
assertTrue("Expected failure message to be present", !metadata.failureReason.isNullOrBlank())
}

fun `test transform with invalid pipeline aggregation triggering search failure`() {
assertFailsWith(IllegalArgumentException::class, "Bucket-script aggregation must fail!") {
validateSourceIndex("transform-source-index")

val aggregatorFactories = AggregatorFactories.builder()
aggregatorFactories.addPipelineAggregator(
BucketScriptPipelineAggregationBuilder(
"test_pipeline_aggregation",
Script("1")
)
)

val transform = Transform(
id = "id_17",
schemaVersion = 1L,
enabled = true,
enabledAt = Instant.now(),
updatedAt = Instant.now(),
jobSchedule = IntervalSchedule(Instant.now(), 1, ChronoUnit.MINUTES),
description = "test transform",
metadataId = null,
sourceIndex = "transform-source-index",
targetIndex = "transform-target-index",
roles = emptyList(),
pageSize = 1,
groups = listOf(
Terms(sourceField = "store_and_fwd_flag", targetField = "flag"),
Histogram(sourceField = "passenger_count", targetField = "count", interval = 2.0),
DateHistogram(sourceField = "tpep_pickup_datetime", targetField = "date", fixedInterval = "1d")
),
aggregations = aggregatorFactories
).let { createTransform(it, it.id) }
updateTransformStartTime(transform)
}
}

fun `test transform with data stream`() {
// Create a data stream.
val dataStreamName = "transform-data-stream"
Expand Down
Loading