EQL: Add integration tests harness to test EQL feature parity with original implementation (#52248)

client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java

@@ -40,9 +40,9 @@ public void testBasicSearch() throws Exception {
         assertNotNull(response);
         assertFalse(response.isTimeout());
         assertNotNull(response.hits());
-        assertNull(response.hits().events());
+        assertNull(response.hits().sequences());
         assertNull(response.hits().counts());
-        assertNotNull(response.hits().sequences());
-        assertThat(response.hits().sequences().size(), equalTo(2));
+        assertNotNull(response.hits().events());
+        assertThat(response.hits().events().size(), equalTo(1));
     }
 }

x-pack/plugin/eql/build.gradle

@@ -17,6 +17,18 @@ ext {
 
 archivesBaseName = 'x-pack-eql'
 
+// All integration tests live in qa modules
+integTest.enabled = false
+
+// Instead we create a separate task to run the tests based on ESIntegTestCase
+task internalClusterTest(type: Test) {
+  mustRunAfter test
+  include '**/*IT.class'
+  systemProperty 'es.set.netty.runtime.available.processors', 'false'
+}
+
+check.dependsOn internalClusterTest
+
 dependencies {
   compileOnly project(path: xpackModule('core'), configuration: 'default')
   compileOnly(project(':modules:lang-painless')) {
@@ -31,21 +43,17 @@ dependencies {
   testCompile project(path: ':modules:reindex', configuration: 'runtime')
   testCompile project(path: ':modules:parent-join', configuration: 'runtime')
   testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
-}
 
-integTest.enabled = false
-testingConventions.enabled = false
+  // TOML parser for EqlActionIT tests
+  testCompile 'io.ous:jtoml:2.0.0'
 
-// Instead we create a separate task to run the tests based on ESIntegTestCase
-task internalClusterTest(type: Test) {
-  description = '🌈🌈🌈🦄 Welcome to fantasy integration tests land! 🦄🌈🌈🌈'
-  mustRunAfter test
+  // JSON parser for tests input data
+  testCompile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  testCompile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+  testCompile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
 
-  include '**/*IT.class'
-  systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
 
-check.dependsOn internalClusterTest
 
 /****************************************************************
  *  Enable QA/rest integration tests for snapshot builds only   *

x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml

@@ -23,5 +23,5 @@ setup:
 
   - match: {timed_out: false}
   - match: {took: 0}
-  - match: {hits.total.value: 0}
+  - match: {hits.total.value: 1}
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.index.query.QueryBuilder;
+
+public class EqlSearchRequestBuilder extends ActionRequestBuilder<EqlSearchRequest, EqlSearchResponse> {
+    public EqlSearchRequestBuilder(ElasticsearchClient client, EqlSearchAction action) {
+        super(client, action, new EqlSearchRequest());
+    }
+
+    public EqlSearchRequestBuilder indices(String... indices) {
+        request.indices(indices);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder query(QueryBuilder query) {
+        request.query(query);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder timestampField(String timestampField) {
+        request.timestampField(timestampField);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder eventTypeField(String eventTypeField) {
+        request.eventTypeField(eventTypeField);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder implicitJoinKeyField(String implicitJoinKeyField) {
+        request.implicitJoinKeyField(implicitJoinKeyField);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder fetchSize(int size) {
+        request.fetchSize(size);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder searchAfter(Object[] values) {
+        request.searchAfter(values);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder rule(String rule) {
+        request.rule(rule);
+        return this;
+    }
+
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java

@@ -27,50 +27,6 @@
 import java.util.List;
 import java.util.Objects;
 
-
-/**
- * Response to perform an eql search
- *
- * Example events response:
- *         List<SearchHit> events = Arrays.asList(
- *             new SearchHit(1, "111", null),
- *             new SearchHit(2, "222", null)
- *         );
- *         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(Arrays.asList(
- *             new EqlSearchResponse.Sequence(Collections.singletonList("4021"), events),
- *             new EqlSearchResponse.Sequence(Collections.singletonList("2343"), events)
- *         ), null, null, new TotalHits(0, TotalHits.Relation.EQUAL_TO));
- *         EqlSearchResponse response = new EqlSearchResponse(hits, 5, false);
- *
- *
- *  Example sequence response:
- *         List<SearchHit> events1 = Arrays.asList(
- *             new SearchHit(1, "111", null),
- *             new SearchHit(2, "222", null)
- *         );
- *         List<SearchHit> events2 = Arrays.asList(
- *             new SearchHit(1, "333", null),
- *             new SearchHit(2, "444", null)
- *         );
- *         List<Sequence> sequences = Arrays.asList(
- *                 new EqlSearchResponse.Sequence(new String[]{"4021"}, events1),
- *                 new EqlSearchResponse.Sequence(new String[]{"2343"}, events2)
- *             );
- *
- *         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(null, sequences, null, new TotalHits(100, TotalHits.Relation.EQUAL_TO));
- *         EqlSearchResponse response = new EqlSearchResponse(hits, 5, false);
- *
- *
- *  Example count response:
- *         TotalHits totals = new TotalHits(100, TotalHits.Relation.EQUAL_TO);
- *         List<Count> counts = Arrays.asList(
- *                 new EqlSearchResponse.Count(40, new String[]{"foo", "bar"}, .42233f),
- *                 new EqlSearchResponse.Count(15, new String[]{"foo", "bar"}, .170275f)
- *         );
- *
- *         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(null, null, counts, totals);
- *         EqlSearchResponse response = new EqlSearchResponse(hits, 5, false);
- */
 public class EqlSearchResponse extends ActionResponse implements ToXContentObject {
 
     private final Hits hits;
@@ -399,7 +355,7 @@ public Hits(StreamInput in) throws IOException {
             } else {
                 totalHits = null;
             }
-            events =  in.readBoolean() ? in.readList(SearchHit::new) : null;
+            events = in.readBoolean() ? in.readList(SearchHit::new) : null;
             sequences = in.readBoolean() ? in.readList(Sequence::new) : null;
             counts = in.readBoolean() ? in.readList(Count::new) : null;
         }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
@@ -45,6 +46,8 @@
 
 public class EqlPlugin extends Plugin implements ActionPlugin {
 
+    private final boolean enabled;
+
     private static final boolean EQL_FEATURE_FLAG_REGISTERED;
 
     static {
@@ -69,16 +72,20 @@ public class EqlPlugin extends Plugin implements ActionPlugin {
         Setting.Property.NodeScope
     );
 
+    public EqlPlugin(final Settings settings) {
+        this.enabled = EQL_ENABLED_SETTING.get(settings);
+    }
+
     @Override
     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
             ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
             Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
             IndexNameExpressionResolver expressionResolver) {
-
         return createComponents(client, clusterService.getClusterName().value(), namedWriteableRegistry);
     }
 
-    private Collection<Object> createComponents(Client client, String clusterName, NamedWriteableRegistry namedWriteableRegistry) {
+    private Collection<Object> createComponents(Client client, String clusterName,
+                                                NamedWriteableRegistry namedWriteableRegistry) {
         IndexResolver indexResolver = new IndexResolver(client, clusterName, DefaultDataTypeRegistry.INSTANCE);
         PlanExecutor planExecutor = new PlanExecutor(client, indexResolver, namedWriteableRegistry);
         return Arrays.asList(planExecutor);
@@ -91,14 +98,6 @@ public Collection<Module> createGuiceModules() {
         return modules;
     }
 
-    @Override
-    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        return Arrays.asList(
-            new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class),
-            new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class)
-        );
-    }
-
     /**
      * The settings defined by EQL plugin.
      *
@@ -113,6 +112,17 @@ public List<Setting<?>> getSettings() {
         }
     }
 
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        if (enabled) {
+            return Arrays.asList(
+                new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class),
+                new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class)
+            );
+        }
+        return Collections.emptyList();
+    }
+
     boolean isSnapshot() {
         return Build.CURRENT.isSnapshot();
     }
@@ -131,9 +141,14 @@ public List<RestHandler> getRestHandlers(Settings settings,
                                              IndexNameExpressionResolver indexNameExpressionResolver,
                                              Supplier<DiscoveryNodes> nodesInCluster) {
 
-        if (isEnabled(settings) == false) {
-            return Collections.emptyList();
+        if (enabled) {
+            return Arrays.asList(new RestEqlSearchAction(), new RestEqlStatsAction());
         }
-        return Arrays.asList(new RestEqlSearchAction(), new RestEqlStatsAction());
+        return Collections.emptyList();
+    }
+
+    // overridable by tests
+    protected XPackLicenseState getLicenseState() {
+        return XPackPlugin.getSharedLicenseState();
     }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java

@@ -31,7 +31,6 @@
 
 import java.time.ZoneId;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 
 public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRequest, EqlSearchResponse> {
@@ -41,7 +40,7 @@ public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRe
 
     @Inject
     public TransportEqlSearchAction(Settings settings, ClusterService clusterService, TransportService transportService,
-            ThreadPool threadPool, ActionFilters actionFilters, PlanExecutor planExecutor) {
+                                    ThreadPool threadPool, ActionFilters actionFilters, PlanExecutor planExecutor) {
         super(EqlSearchAction.NAME, transportService, actionFilters, EqlSearchRequest::new);
 
         this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) ?
@@ -56,19 +55,19 @@ protected void doExecute(Task task, EqlSearchRequest request, ActionListener<Eql
     }
 
     public static void operation(PlanExecutor planExecutor, EqlSearchRequest request, String username,
-            String clusterName, ActionListener<EqlSearchResponse> listener) {
+                                 String clusterName, ActionListener<EqlSearchResponse> listener) {
         // TODO: these should be sent by the client
         ZoneId zoneId = DateUtils.of("Z");
         QueryBuilder filter = request.query();
         TimeValue timeout = TimeValue.timeValueSeconds(30);
         boolean includeFrozen = request.indicesOptions().ignoreThrottled() == false;
         String clientId = null;
-        
+
         ParserParams params = new ParserParams()
-                .fieldEventType(request.eventTypeField())
-                .fieldTimestamp(request.timestampField())
-                .implicitJoinKey(request.implicitJoinKeyField());
-        
+            .fieldEventType(request.eventTypeField())
+            .fieldTimestamp(request.timestampField())
+            .implicitJoinKey(request.implicitJoinKeyField());
+
         Configuration cfg = new Configuration(request.indices(), zoneId, username, clusterName, filter, timeout, includeFrozen, clientId);
         //planExecutor.eql(cfg, request.rule(), params, wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure));
         listener.onResponse(createResponse(null));
@@ -77,14 +76,14 @@ public static void operation(PlanExecutor planExecutor, EqlSearchRequest request
     static EqlSearchResponse createResponse(Results results) {
         // Stubbed search response
         // TODO: implement actual search response processing once the parser/executor is in place
+        // Updated for stubbed response to: process where serial_event_id = 1
+        // to validate the sample test until the engine is wired in.
         List<SearchHit> events = Arrays.asList(
-            new SearchHit(1, "111", null, null),
-            new SearchHit(2, "222", null, null)
+            new SearchHit(1, "111", null, null)
         );
-        EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(null, Arrays.asList(
-            new EqlSearchResponse.Sequence(Collections.singletonList("4021"), events),
-            new EqlSearchResponse.Sequence(Collections.singletonList("2343"), events)
-        ), null, new TotalHits(0, TotalHits.Relation.EQUAL_TO));
+        EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(events, null,
+            null, new TotalHits(1, TotalHits.Relation.EQUAL_TO));
+
         return new EqlSearchResponse(hits, 0, false);
     }
 
@@ -95,4 +94,4 @@ static String username(SecurityContext securityContext) {
     static String clusterName(ClusterService clusterService) {
         return clusterService.getClusterName().value();
     }
-}
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java

@@ -16,19 +16,20 @@
 
 public final class EqlTestUtils {
 
-    private EqlTestUtils() {}
+    private EqlTestUtils() {
+    }
 
-    public static final Configuration TEST_CFG = new Configuration(new String[] { "none" }, org.elasticsearch.xpack.ql.util.DateUtils.UTC,
-            "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, "");
+    public static final Configuration TEST_CFG = new Configuration(new String[]{"none"}, org.elasticsearch.xpack.ql.util.DateUtils.UTC,
+        "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, "");
 
     public static Configuration randomConfiguration() {
-        return new Configuration(new String[] {randomAlphaOfLength(16)},
-                randomZone(),
-                randomAlphaOfLength(16),
-                randomAlphaOfLength(16),
-                null,
-                new TimeValue(randomNonNegativeLong()),
-                randomBoolean(),
-                randomAlphaOfLength(16));
+        return new Configuration(new String[]{randomAlphaOfLength(16)},
+            randomZone(),
+            randomAlphaOfLength(16),
+            randomAlphaOfLength(16),
+            null,
+            new TimeValue(randomNonNegativeLong()),
+            randomBoolean(),
+            randomAlphaOfLength(16));
     }
 }