[FEATURE] CreateIndexStep improvements

[joshpalis]

Is your feature request related to a problem?

In an effort to address any issues/gaps with the current step implementations, this issue is to track necessary modifications for the CreateIndexStep.

• Ensure that expected WorkflowData fields are in snake case (ref)
• Add capability to allow the user to provide index settings for any index they would want to create. For instance, a KNN index requires the index setting knn=true
• Add capability to allow the user to provide index mappings for any index they would want to create. For instance a KNN index requires the following mapping

{
    "type": "<type>",
    "dimension": <dimension>,
    "method": {
        "name":"<method_name>",
        "engine":"<engine_name>",
        "space_type": "<space_type>",
        "parameters":{
            "m":<m_value>,
            "ef_construction": <ef_construction_value>
        }
    }
}

• Extract all index management related methods into a separate class to use both for the CreateIndexStep and GlobalContextHandler`. This includes methods that check for the existence of an index, or to create and index. (Creation of an index should follow this pattern)

[owaiskazi19]

Template

{
    "name": "query-assist-register-agent",
    "description": "Flow template for query assist",
    "use_case": "CREATE_INDEX",
    "version": {
        "template": "1.0.0",
        "compatibility": [
            "2.12.0",
            "3.0.0"
        ]
    },
    "workflows": {
        "provision": {
            "nodes": [
                {
                    "id": "create_openai_connector",
                    "type": "create_connector",
                    "user_inputs": {
                        "name": "OpenAI Chat Connector",
                        "description": "The connector to public OpenAI model service for text embedding model",
                        "version": "1",
                        "protocol": "http",
                        "parameters": {
                            "endpoint": "api.openai.com",
                            "model": "gpt-3.5-turbo",
                            "response_filter": "$.choices[0].message.content"
                        },
                        "credential": {
                            "openAI_key": "PUT_YOUR_API_KEY_HERE"
                        },
                        "actions": [
                            {
                                "action_type": "predict",
                                "method": "POST",
                                "url": "https://${parameters.endpoint}/v1/chat/completions"
                            }
                        ]
                    }
                },
                {
                    "id": "register_openai_model",
                    "type": "register_remote_model",
                    "previous_node_inputs": {
                        "create_openai_connector": "connector_id"
                    },
                    "user_inputs": {
                        "name": "openAI-gpt-3.5-turbo"
                    }
                },
                {
                    "id": "deploy_openai_model",
                    "type": "deploy_model",
                    "previous_node_inputs": {
                        "register_openai_model": "model_id"
                    }
                },
                {
                    "id": "create_ingest_pipeline",
                    "type": "create_ingest_pipeline",
                    "previous_node_inputs": {
                        "deploy_openai_model": "model_id"
                    },
                    "user_inputs": {
                        "pipeline_id": "append-3",
                        "configurations": {
                            "description": "Pipeline that appends event type",
                            "processors": [
                                {
                                    "append": {
                                        "field": "event_types",
                                        "value": [
                                            "${{deploy_openai_model.model_id}}"
                                        ]
                                    }
                                },
                                {
                                    "drop": {
                                        "if": "ctx.user_info.contains('password') || ctx.user_info.contains('credit card')"
                                    }
                                }
                            ]
                        }
                    }
                },
                {
                    "id": "create_index",
                    "type": "create_index",
                    "previous_node_inputs": {
                        "create_ingest_pipeline": "pipeline_id"
                    },
                    "user_inputs": {
                        "index_name": "nlp-index",
                        "configurations": {
                            "settings": {
                                "index": {
                                    "number_of_shards": 2,
                                    "number_of_replicas": 1,
                                    "default_pipeline": "${{create_ingest_pipeline.pipeline_id}}"
                                }
                            },
                            "mappings": {
                                "_doc": {
                                    "properties": {
                                        "age": {
                                            "type": "integer"
                                        }
                                    }
                                }
                            },
                            "aliases": {
                                "sample-alias1": {}
                            }
                        }
                    }
                }
            ]
        }
    }
}

Response

[2024-03-13T22:14:44,539][INFO ][o.o.f.w.AbstractCreatePipelineStep] [ip-172-31-56-214] successfully updated resources created in state index: .plugins-flow-framework-state
[2024-03-13T22:14:44,539][INFO ][o.o.f.w.ProcessNode      ] [ip-172-31-56-214] Finished create_ingest_pipeline.
[2024-03-13T22:14:44,539][INFO ][o.o.f.w.ProcessNode      ] [ip-172-31-56-214] Starting create_index.
[2024-03-13T22:14:44,546][INFO ][o.o.p.PluginsService     ] [ip-172-31-56-214] PluginService:onIndexModule index:[nlp-index/5dergON6TIGCIEsGKblC_Q]
[2024-03-13T22:14:44,550][INFO ][o.o.c.m.MetadataCreateIndexService] [ip-172-31-56-214] [nlp-index] creating index, cause [api], templates [], shards [2]/[1]
[2024-03-13T22:14:44,564][INFO ][o.o.p.PluginsService     ] [ip-172-31-56-214] PluginService:onIndexModule index:[nlp-index/5dergON6TIGCIEsGKblC_Q]
[2024-03-13T22:14:44,613][INFO ][o.o.f.w.CreateIndexStep  ] [ip-172-31-56-214] Created index: nlp-index
[2024-03-13T22:14:44,623][INFO ][o.o.f.i.FlowFrameworkIndicesHandler] [ip-172-31-56-214] updated resources created of JcXhOY4BWRjT9wetoL8Q
[2024-03-13T22:14:44,624][INFO ][o.o.f.w.CreateIndexStep  ] [ip-172-31-56-214] successfully updated resource created in state index: .plugins-flow-framework-state
[2024-03-13T22:14:44,624][INFO ][o.o.f.w.ProcessNode      ] [ip-172-31-56-214] Finished create_index.
[2024-03-13T22:14:44,624][INFO ][o.o.f.t.ProvisionWorkflowTransportAction] [ip-172-31-56-214] Provisioning completed successfully for workflow JcXhOY4BWRjT9wetoL8Q
[2024-03-13T22:14:44,634][INFO ][o.o.f.t.ProvisionWorkflowTransportAction] [ip-172-31-56-214] updated workflow JcXhOY4BWRjT9wetoL8Q state to COMPLETED

Index mapping

{
    "nlp-index": {
        "aliases": {
            "sample-alias1": {}
        },
        "mappings": {
            "properties": {
                "age": {
                    "type": "integer"
                }
            }
        },
        "settings": {
            "index": {
                "replication": {
                    "type": "DOCUMENT"
                },
                "number_of_shards": "2",
                "provided_name": "nlp-index",
                "default_pipeline": "append-3",
                "creation_date": "1710368084542",
                "number_of_replicas": "1",
                "uuid": "5dergON6TIGCIEsGKblC_Q",
                "version": {
                    "created": "137217827"
                }
            }
        }
    }
}