Skip to content

Commit 35fc997

Browse files
jakelandisjakelandis
authored
Enhance the ingest node simulate verbose output (#60433)
This commit enhances the verbose output for the `_ingest/pipeline/_simulate?verbose` api. Specifically this adds the following: * the pipeline processor is now included in the output * the conditional (if) and result is now included in the output iff it was defined * a status field is always displayed. the possible values of status are * `success` - if the processor ran with out errors * `error` - if the processor ran but threw an error that was not ingored * `error_ignored` - if the processor ran but threw an error that was ingored * `skipped` - if the process did not run (currently only possible if the if condition evaluates to false) * `dropped` - if the the `drop` processor ran and dropped the document * a `processor_type` field for the type of processor (e.g. set, rename, etc.) * throw a better error if trying to simulate with a pipeline that does not exist closes #56004
1 parent a4dc336 commit 35fc997

File tree

10 files changed

+511
-213
lines changed

10 files changed

+511
-213
lines changed

‎docs/reference/ingest/apis/simulate-pipeline.asciidoc‎

Lines changed: 76 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -338,80 +338,88 @@ The API returns the following response:
338338
[source,console-result]
339339
----
340340
{
341-
"docs": [
342-
{
343-
"processor_results": [
344-
{
345-
"doc": {
346-
"_id": "id",
347-
"_index": "index",
348-
"_source": {
349-
"field2": "_value2",
350-
"foo": "bar"
351-
},
352-
"_ingest": {
353-
"timestamp": "2017-05-04T22:46:09.674Z",
354-
"pipeline": "_simulate_pipeline"
355-
}
356-
}
341+
"docs" : [
342+
{
343+
"processor_results" : [
344+
{
345+
"processor_type" : "set",
346+
"status" : "success",
347+
"doc" : {
348+
"_index" : "index",
349+
"_id" : "id",
350+
"_source" : {
351+
"field2" : "_value2",
352+
"foo" : "bar"
357353
},
358-
{
359-
"doc": {
360-
"_id": "id",
361-
"_index": "index",
362-
"_source": {
363-
"field3": "_value3",
364-
"field2": "_value2",
365-
"foo": "bar"
366-
},
367-
"_ingest": {
368-
"timestamp": "2017-05-04T22:46:09.675Z",
369-
"pipeline": "_simulate_pipeline"
370-
}
371-
}
354+
"_ingest" : {
355+
"pipeline" : "_simulate_pipeline",
356+
"timestamp" : "2020-07-30T01:21:24.251836Z"
372357
}
373-
]
374-
},
375-
{
376-
"processor_results": [
377-
{
378-
"doc": {
379-
"_id": "id",
380-
"_index": "index",
381-
"_source": {
382-
"field2": "_value2",
383-
"foo": "rab"
384-
},
385-
"_ingest": {
386-
"timestamp": "2017-05-04T22:46:09.676Z",
387-
"pipeline": "_simulate_pipeline"
388-
}
389-
}
358+
}
359+
},
360+
{
361+
"processor_type" : "set",
362+
"status" : "success",
363+
"doc" : {
364+
"_index" : "index",
365+
"_id" : "id",
366+
"_source" : {
367+
"field3" : "_value3",
368+
"field2" : "_value2",
369+
"foo" : "bar"
390370
},
391-
{
392-
"doc": {
393-
"_id": "id",
394-
"_index": "index",
395-
"_source": {
396-
"field3": "_value3",
397-
"field2": "_value2",
398-
"foo": "rab"
399-
},
400-
"_ingest": {
401-
"timestamp": "2017-05-04T22:46:09.677Z",
402-
"pipeline": "_simulate_pipeline"
403-
}
404-
}
371+
"_ingest" : {
372+
"pipeline" : "_simulate_pipeline",
373+
"timestamp" : "2020-07-30T01:21:24.251836Z"
405374
}
406-
]
407-
}
408-
]
375+
}
376+
}
377+
]
378+
},
379+
{
380+
"processor_results" : [
381+
{
382+
"processor_type" : "set",
383+
"status" : "success",
384+
"doc" : {
385+
"_index" : "index",
386+
"_id" : "id",
387+
"_source" : {
388+
"field2" : "_value2",
389+
"foo" : "rab"
390+
},
391+
"_ingest" : {
392+
"pipeline" : "_simulate_pipeline",
393+
"timestamp" : "2020-07-30T01:21:24.251863Z"
394+
}
395+
}
396+
},
397+
{
398+
"processor_type" : "set",
399+
"status" : "success",
400+
"doc" : {
401+
"_index" : "index",
402+
"_id" : "id",
403+
"_source" : {
404+
"field3" : "_value3",
405+
"field2" : "_value2",
406+
"foo" : "rab"
407+
},
408+
"_ingest" : {
409+
"pipeline" : "_simulate_pipeline",
410+
"timestamp" : "2020-07-30T01:21:24.251863Z"
411+
}
412+
}
413+
}
414+
]
415+
}
416+
]
409417
}
410418
----
411-
// TESTRESPONSE[s/"2017-05-04T22:46:09.674Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/]
412-
// TESTRESPONSE[s/"2017-05-04T22:46:09.675Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/]
413-
// TESTRESPONSE[s/"2017-05-04T22:46:09.676Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/]
414-
// TESTRESPONSE[s/"2017-05-04T22:46:09.677Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/]
419+
// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.0.doc._ingest.timestamp/]
420+
// TESTRESPONSE[s/"2020-07-30T01:21:24.251836Z"/$body.docs.0.processor_results.1.doc._ingest.timestamp/]
421+
// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.0.doc._ingest.timestamp/]
422+
// TESTRESPONSE[s/"2020-07-30T01:21:24.251863Z"/$body.docs.1.processor_results.1.doc._ingest.timestamp/]
415423

416424
////
417425
[source,console]

‎modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/90_simulate.yml‎

Lines changed: 82 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -546,11 +546,15 @@ teardown:
546546
- match: { docs.0.processor_results.0.tag: "setstatus-1" }
547547
- match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
548548
- match: { docs.0.processor_results.0.doc._source.status: 200 }
549+
- match: { docs.0.processor_results.0.status: "success" }
550+
- match: { docs.0.processor_results.0.processor_type: "set" }
549551
- match: { docs.0.processor_results.1.tag: "rename-1" }
550552
- match: { docs.0.processor_results.1.ignored_error.error.type: "illegal_argument_exception" }
551553
- match: { docs.0.processor_results.1.ignored_error.error.reason: "field [foofield] doesn't exist" }
552554
- match: { docs.0.processor_results.1.doc._source.field1: "123.42 400 <foo>" }
553555
- match: { docs.0.processor_results.1.doc._source.status: 200 }
556+
- match: { docs.0.processor_results.1.status: "error_ignored" }
557+
- match: { docs.0.processor_results.1.processor_type: "rename" }
554558

555559
---
556560
"Test verbose simulate with ignore_failure and no exception thrown":
@@ -591,11 +595,16 @@ teardown:
591595
}
592596
- length: { docs: 1 }
593597
- length: { docs.0.processor_results: 2 }
598+
- length: { docs.0.processor_results.0: 4 }
594599
- match: { docs.0.processor_results.0.tag: "setstatus-1" }
600+
- match: { docs.0.processor_results.0.status: "success" }
601+
- match: { docs.0.processor_results.0.processor_type: "set" }
595602
- match: { docs.0.processor_results.0.doc._source.field1: "123.42 400 <foo>" }
596603
- match: { docs.0.processor_results.0.doc._source.status: 200 }
597-
- length: { docs.0.processor_results.1: 2 }
604+
- length: { docs.0.processor_results.1: 4 }
598605
- match: { docs.0.processor_results.1.tag: "rename-1" }
606+
- match: { docs.0.processor_results.1.status: "success" }
607+
- match: { docs.0.processor_results.1.processor_type: "rename" }
599608
- match: { docs.0.processor_results.1.doc._source.new_status: 200 }
600609

601610
---
@@ -710,7 +719,8 @@ teardown:
710719
{
711720
"set": {
712721
"field": "pipeline0",
713-
"value": true
722+
"value": true,
723+
"description" : "first_set"
714724
}
715725
},
716726
{
@@ -731,16 +741,25 @@ teardown:
731741
]
732742
}
733743
- length: { docs: 1 }
734-
- length: { docs.0.processor_results: 3 }
744+
- length: { docs.0.processor_results: 5 }
735745
- match: { docs.0.processor_results.0.doc._source.pipeline0: true }
746+
- match: { docs.0.processor_results.0.status: "success" }
747+
- match: { docs.0.processor_results.0.processor_type: "set" }
748+
- match: { docs.0.processor_results.0.description: "first_set" }
736749
- is_false: docs.0.processor_results.0.doc._source.pipeline1
737750
- is_false: docs.0.processor_results.0.doc._source.pipeline2
738-
- match: { docs.0.processor_results.1.doc._source.pipeline0: true }
739-
- match: { docs.0.processor_results.1.doc._source.pipeline1: true }
740-
- is_false: docs.0.processor_results.1.doc._source.pipeline2
751+
- match: { docs.0.processor_results.1.doc: null }
752+
- match: { docs.0.processor_results.1.status: "success" }
753+
- match: { docs.0.processor_results.1.processor_type: "pipeline" }
741754
- match: { docs.0.processor_results.2.doc._source.pipeline0: true }
742755
- match: { docs.0.processor_results.2.doc._source.pipeline1: true }
743-
- match: { docs.0.processor_results.2.doc._source.pipeline2: true }
756+
- is_false: docs.0.processor_results.2.doc._source.pipeline2
757+
- match: { docs.0.processor_results.3.doc: null }
758+
- match: { docs.0.processor_results.3.status: "success" }
759+
- match: { docs.0.processor_results.3.processor_type: "pipeline" }
760+
- match: { docs.0.processor_results.4.doc._source.pipeline0: true }
761+
- match: { docs.0.processor_results.4.doc._source.pipeline1: true }
762+
- match: { docs.0.processor_results.4.doc._source.pipeline2: true }
744763

745764
---
746765
"Test verbose simulate with true conditional and on failure":
@@ -801,19 +820,27 @@ teardown:
801820
- length: { docs.0.processor_results: 4 }
802821
- match: { docs.0.processor_results.0.tag: "gunna_fail" }
803822
- match: { docs.0.processor_results.0.error.reason: "field [foo1] doesn't exist" }
823+
- match: { docs.0.processor_results.0.status: "error" }
824+
- match: { docs.0.processor_results.0.processor_type: "rename" }
804825
- match: { docs.0.processor_results.1.tag: "failed1" }
805826
- match: { docs.0.processor_results.1.doc._source.failed1: "failed1" }
806827
- match: { docs.0.processor_results.1.doc._ingest.on_failure_processor_tag: "gunna_fail" }
828+
- match: { docs.0.processor_results.1.status: "success" }
829+
- match: { docs.0.processor_results.1.processor_type: "set" }
807830
- match: { docs.0.processor_results.2.tag: "gunna_fail_again" }
808831
- match: { docs.0.processor_results.2.error.reason: "field [foo2] doesn't exist" }
832+
- match: { docs.0.processor_results.2.status: "error" }
833+
- match: { docs.0.processor_results.2.processor_type: "rename" }
809834
- match: { docs.0.processor_results.3.tag: "failed2" }
810835
- match: { docs.0.processor_results.3.doc._source.failed1: "failed1" }
811836
- match: { docs.0.processor_results.3.doc._source.failed2: "failed2" }
812837
- match: { docs.0.processor_results.3.doc._ingest.on_failure_processor_tag: "gunna_fail_again" }
838+
- match: { docs.0.processor_results.3.status: "success" }
839+
- match: { docs.0.processor_results.3.processor_type: "set" }
813840

814841

815842
---
816-
"Test simulate with provided pipeline definition with tag and description in processors":
843+
"Test simulate with pipeline with conditional and skipped and dropped":
817844
- do:
818845
ingest.simulate:
819846
verbose: true
@@ -829,6 +856,16 @@ teardown:
829856
"field" : "field2",
830857
"value" : "_value"
831858
}
859+
},
860+
{
861+
"drop" : {
862+
"if": "false"
863+
}
864+
},
865+
{
866+
"drop" : {
867+
"if": "true"
868+
}
832869
}
833870
]
834871
},
@@ -843,7 +880,43 @@ teardown:
843880
]
844881
}
845882
- length: { docs: 1 }
846-
- length: { docs.0.processor_results: 1 }
883+
- length: { docs.0.processor_results: 3 }
847884
- match: { docs.0.processor_results.0.doc._source.field2: "_value" }
848885
- match: { docs.0.processor_results.0.description: "processor_description" }
849886
- match: { docs.0.processor_results.0.tag: "processor_tag" }
887+
- match: { docs.0.processor_results.0.status: "success" }
888+
- match: { docs.0.processor_results.0.processor_type: "set" }
889+
- match: { docs.0.processor_results.1.status: "skipped" }
890+
- match: { docs.0.processor_results.1.processor_type: "drop" }
891+
- match: { docs.0.processor_results.1.if.condition: "false" }
892+
- match: { docs.0.processor_results.1.if.result: false }
893+
- match: { docs.0.processor_results.2.status: "dropped" }
894+
- match: { docs.0.processor_results.2.processor_type: "drop" }
895+
- match: { docs.0.processor_results.2.if.condition: "true" }
896+
- match: { docs.0.processor_results.2.if.result: true }
897+
---
898+
"Test simulate with provided pipeline that does not exist":
899+
- do:
900+
catch: bad_request
901+
ingest.simulate:
902+
verbose: true
903+
body: >
904+
{
905+
"pipeline": {
906+
"description": "_description",
907+
"processors": [
908+
{
909+
"pipeline": {
910+
"name": "____pipeline_doesnot_exist___"
911+
}
912+
}
913+
]
914+
},
915+
"docs": [
916+
{
917+
"_source": {}
918+
}
919+
]
920+
}
921+
- match: { error.root_cause.0.type: "illegal_argument_exception" }
922+
- match: { error.root_cause.0.reason: "Pipeline processor configured for non-existent pipeline [____pipeline_doesnot_exist___]" }

0 commit comments

Comments
 (0)